diff --git a/AGENTS.md b/AGENTS.md index a639b6fe6..e3d529006 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ It is kept in sync with `CLAUDE.md`; if you change guidance in one, check whethe EntityFrameworkCore.Jet is an EF Core provider for Microsoft Jet/ACE databases (Microsoft Access `.mdb`/`.accdb` files). The **Jet** provider runs **Windows only** and bridges EF Core to the Access database engine via either ODBC or OLE DB. Alongside it, **LibRed** (also in this repo, on `master`) is a from-scratch managed engine that reads/writes the file format directly and is **cross-platform** — see the LibRed section below. -Current version: `11.0.0-alpha.1` (`Version.props`) targeting EF Core 11 and `net11.0`; `global.json` pins the 11.0.100 RC1 SDK with `rollForward: latestFeature`. The test projects use **xunit v3**. +Current version: `11.0.0-alpha.3` (`Version.props`) targeting EF Core 11 and `net11.0`; `global.json` pins the 11.0.100 RC1 SDK with `rollForward: latestFeature`. The test projects use **xunit v3**. ### Which layer am I touching? @@ -211,7 +211,9 @@ contract. > underneath us. It did: dotnet/runtime#130566 (.NET 11 preview 7) dropped `Convert.ToDecimal`'s 15-significant-digit > rounding, which came from OA's own `VarDecFromR8` and had been stable since the 1990s. That turned > `SUM(ROUND(UnitPrice, 2))` into `58.600000000000001421085471520`. The fix was to own it: -> `src/EFCore.Jet.Data/JetDecimalConverter.cs`. **When a long-stable conversion suddenly misbehaves with no code +> `src/Shared/Data/JetDecimalConverter.cs` (compiled into both EFCore.Jet.Data and LibRed.Core; every LibRed +> assembly uses the Core copy, and any double/float that can reach a decimal goes through its `ToDecimal`, never +> `Convert.ToDecimal` or a cast). **When a long-stable conversion suddenly misbehaves with no code > change on our side, suspect the runtime's OA-era compatibility behaviour before suspecting the provider.** ## LibRed — Native Managed Engine diff --git a/CLAUDE.md b/CLAUDE.md index ae2304379..bebf1608d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co EntityFrameworkCore.Jet is an EF Core provider for Microsoft Jet/ACE databases (Microsoft Access `.mdb`/`.accdb` files). The **Jet** provider runs **Windows only** and bridges EF Core to the Access database engine via either ODBC or OLE DB. Alongside it, **LibRed** (also in this repo, on `master`) is a from-scratch managed engine that reads/writes the file format directly and is **cross-platform** — see the LibRed section below. -Current version: `11.0.0-alpha.1` (`Version.props`) targeting EF Core 11 and `net11.0`; `global.json` pins the 11.0.100 RC1 SDK with `rollForward: latestFeature`. The test projects use **xunit v3**. +Current version: `11.0.0-alpha.3` (`Version.props`) targeting EF Core 11 and `net11.0`; `global.json` pins the 11.0.100 RC1 SDK with `rollForward: latestFeature`. The test projects use **xunit v3**. ### Which layer am I touching? @@ -223,7 +223,9 @@ contract. > underneath us. It did: dotnet/runtime#130566 (.NET 11 preview 7) dropped `Convert.ToDecimal`'s 15-significant-digit > rounding, which came from OA's own `VarDecFromR8` and had been stable since the 1990s. That turned > `SUM(ROUND(UnitPrice, 2))` into `58.600000000000001421085471520`. The fix was to own it: -> `src/EFCore.Jet.Data/JetDecimalConverter.cs`. **When a long-stable conversion suddenly misbehaves with no code +> `src/Shared/Data/JetDecimalConverter.cs` (compiled into both EFCore.Jet.Data and LibRed.Core; every LibRed +> assembly uses the Core copy, and any double/float that can reach a decimal goes through its `ToDecimal`, never +> `Convert.ToDecimal` or a cast). **When a long-stable conversion suddenly misbehaves with no code > change on our side, suspect the runtime's OA-era compatibility behaviour before suspecting the provider.** ## LibRed — Native Managed Engine diff --git a/Version.props b/Version.props index 18997bbd6..f7528b9b9 100644 --- a/Version.props +++ b/Version.props @@ -17,7 +17,7 @@ --> 11.0.0 alpha - 2 + 3 + + + diff --git a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs index 1e2e09774..3bbcd0187 100644 --- a/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs +++ b/src/LibRed/LibRed.Core/Pages/DatabaseDefinitionPage.cs @@ -47,6 +47,14 @@ public sealed class DatabaseDefinitionPage : Page /// pointer at . 2 in every observed file. public int CatalogRootPage { get; internal set; } + /// Where the global free-pages usage map lives, from the [row:1][page:3] pointer at + /// . Page 1 row 0 in every file ACE writes. + public (int Row, int Page) FreePagesMap { get; internal set; } + + /// Where the global released-pages usage map lives, from the [row:1][page:3] pointer at + /// . Page 1 row 1 in every file ACE writes. + public (int Row, int Page) ReleasedPagesMap { get; internal set; } + public DateTime DatabaseCreationDate { get; internal set; } public override void Read(PageBuffer buffer, Formats.JetFormatBase format) @@ -67,6 +75,8 @@ public override void Read(PageBuffer buffer, Formats.JetFormatBase format) DefaultCollationSortId = clear[Formats.JetFormatBase.CollationSortIdOffset - b]; DefaultCollationVersion = clear[Formats.JetFormatBase.CollationVersionOffset - b]; CatalogRootPage = BinaryPrimitives.ReadInt32LittleEndian(clear.Slice(Formats.JetFormatBase.CatalogRootPointerOffset - b, 4)); + FreePagesMap = ReadMapPointer(buffer.Span, Formats.JetFormatBase.FreePagesMapPointerOffset); + ReleasedPagesMap = ReadMapPointer(buffer.Span, Formats.JetFormatBase.ReleasedPagesMapPointerOffset); // An OLE Automation date, so it is decoded by the OA function rather than by hand: the two disagree // below the epoch, where OA keeps the time fraction positive (-1.25 is 1899-12-29 06:00, not // 1899-12-28 18:00). And the value comes straight off page 0, so a NaN, an infinity or anything past @@ -79,6 +89,18 @@ public override void Read(PageBuffer buffer, Formats.JetFormatBase format) DatabaseCreationDate = DateTime.FromOADate(days); } + /// Decodes one of page 0's global usage-map pointers: a masked little-endian word whose low byte is + /// the record's row and whose upper three bytes are its page. + internal static (int Row, int Page) ReadMapPointer(ReadOnlySpan page, int offset) + { + ReadOnlySpan mask = Formats.JetFormatBase.PageZeroHeaderMask; + int start = Formats.JetFormatBase.PageZeroHeaderMaskStart; + uint value = 0; + for (int i = 0; i < 4; i++) + value |= (uint)(page[offset + i] ^ mask[offset - start + i]) << (8 * i); + return ((int)(value & 0xFF), (int)(value >> 8)); + } + /// XOR-de-obfuscates the page-0 header region into , whose length /// equals the mask length; clear[i] corresponds to page offset /// + i. diff --git a/src/LibRed/LibRed.Core/Pages/PageType.cs b/src/LibRed/LibRed.Core/Pages/PageType.cs index a7e4f6446..9cf9d9fee 100644 --- a/src/LibRed/LibRed.Core/Pages/PageType.cs +++ b/src/LibRed/LibRed.Core/Pages/PageType.cs @@ -12,4 +12,19 @@ public enum PageType : byte IntermediateIndexPage = 0x03, LeafIndexPage = 0x04, PageUsageBitmap = 0x05, + + /// A table-definition page that has been released by DROP TABLE. Access marks it by + /// setting this type byte and changing nothing else — the old definition stays on the page — and it + /// leaves the data, long-value and usage-map holder pages it frees at their original types. Measured: + /// exactly one byte of the 4,096 differs across an ACE drop. Compact reclaims the page. + /// See docs/format/page-08-released-tdef.md. + ReleasedTableDefinition = 0x08, + + /// A long-value page released because the last value sharing it was deleted. Several small + /// (single-page form) values pack onto one LVAL page; each delete retires its row to a 0-length + /// deleted+overflow tombstone, and when none are left the page is stamped with this type and freed. + /// A chained value owns its pages outright and they go back at instead, + /// which is why only the packed form produces this. Nothing needs to handle it on read: allocation + /// selects on the free map, not on this byte. See docs/format/page-09-released-long-value.md. + ReleasedLongValuePage = 0x09, } diff --git a/src/LibRed/LibRed.Core/Pages/TdefChainReader.cs b/src/LibRed/LibRed.Core/Pages/TdefChainReader.cs index 9df637367..b6954bb84 100644 --- a/src/LibRed/LibRed.Core/Pages/TdefChainReader.cs +++ b/src/LibRed/LibRed.Core/Pages/TdefChainReader.cs @@ -28,11 +28,15 @@ internal static (PageBuffer Buffer, IReadOnlyList ContinuationPages) Read( $"TDEF page {firstPage} declares length {definitionLength}; supported validated range is " + $"{format.TdefRealIndexBlockOffset} through {MaxDefinitionLength} bytes."); + // The chain holds the definition AND its 8-byte trailing reserve, which follows the last definition byte + // and spills onto a page of its own when it does not fit — so a continuation page can carry no definition + // bytes at all (verified vs ACE: a 4,090-byte definition has a continuation holding two reserve bytes). int pageSize = format.PageSize; int bodySize = pageSize - JetFormatBase.TdefContinuationHeaderSize; - int continuationCount = definitionLength <= pageSize + int stored = definitionLength + JetFormatBase.TdefContinuationHeaderSize; + int continuationCount = stored <= pageSize ? 0 - : (definitionLength - pageSize + bodySize - 1) / bodySize; + : (stored - pageSize + bodySize - 1) / bodySize; int next = first.ReadInt32(format.TdefNextPageOffset); var continuationPages = new List(continuationCount); diff --git a/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedEvaluator.cs b/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedEvaluator.cs index 0e80a2c48..9c6475e80 100644 --- a/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedEvaluator.cs +++ b/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedEvaluator.cs @@ -1,4 +1,5 @@ using System.Globalization; +using EntityFrameworkCore.Jet.Data; using LibRed.Catalog; namespace LibRed.Storage.Calculated; @@ -574,7 +575,7 @@ private static bool Like(string text, string pattern) JetDataType.Int64 => (long)Math.Round(ToNumber(value), MidpointRounding.ToEven), JetDataType.Single => (float)ToNumber(value), JetDataType.Double => ToNumber(value), - JetDataType.Currency => Math.Round((decimal)ToNumber(value), 4, MidpointRounding.ToEven), + JetDataType.Currency => Math.Round(JetDecimalConverter.FromDouble(ToNumber(value)), 4, MidpointRounding.ToEven), JetDataType.DateTime => ToDate(value), JetDataType.Text or JetDataType.Memo => ToText(value), _ => throw new CalculatedExpressionException($"Cannot store a calculated {type} value."), diff --git a/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedExpression.cs b/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedExpression.cs index f240eeb61..a7d1dbf4d 100644 --- a/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedExpression.cs +++ b/src/LibRed/LibRed.Core/Storage/Calculated/CalculatedExpression.cs @@ -458,16 +458,18 @@ private static List Tokenise(string text) tokens.Add(new Token(TokenKind.Bracketed, text[(i + 1)..end])); i = end + 1; } - else if (c == '"') + else if (c is '"' or '\'') { + // Text in either quote, a doubled quote standing for itself (verified vs ACE through DAO: + // [A] & 'it''s' & [B] is accepted, stored as written, and gives the same values as "it's"). var value = new System.Text.StringBuilder(); i++; while (true) { if (i >= text.Length) throw new CalculatedExpressionException("Unterminated string in expression."); - if (text[i] == '"') + if (text[i] == c) { - if (i + 1 < text.Length && text[i + 1] == '"') { value.Append('"'); i += 2; continue; } + if (i + 1 < text.Length && text[i + 1] == c) { value.Append(c); i += 2; continue; } i++; break; } diff --git a/src/LibRed/LibRed.Core/Storage/DatabaseCreator.cs b/src/LibRed/LibRed.Core/Storage/DatabaseCreator.cs index 5512f832f..faa0fba3f 100644 --- a/src/LibRed/LibRed.Core/Storage/DatabaseCreator.cs +++ b/src/LibRed/LibRed.Core/Storage/DatabaseCreator.cs @@ -39,14 +39,15 @@ public static byte[] BuildDefinitionPage( string id = isAccdb ? JetFormatBase.AceIdentifier : JetFormatBase.JetIdentifier; Encoding.ASCII.GetBytes(id).CopyTo(page, JetFormatBase.FormatIdentifierOffset); // 0x04, 15 bytes; 0x13 stays NUL page[JetFormatBase.VersionOffset] = version; // 0x14 - page[0x15] = (byte)(version == 0x03 ? 0x01 : 0x00); // 2010-format minor byte + page[JetFormatBase.MinorVersionOffset] = JetFormatBase.CreatedMinorVersion(version); // 0x15 // --- Masked header (0x18..0x97): build the clear image, then XOR the fixed mask over it. --- int b = JetFormatBase.PageZeroHeaderMaskStart; Span clear = stackalloc byte[JetFormatBase.PageZeroHeaderMask.Length]; - BinaryPrimitives.WriteInt32LittleEndian(clear[(0x18 - b)..], 0x00000100); // 0x18 fixed constant - BinaryPrimitives.WriteInt32LittleEndian(clear[(0x1C - b)..], 0x00000101); // 0x1C fixed constant + // 0x18/0x1C: [row][page] pointers to the global usage maps on page 1 — free pages (row 0), released pages (row 1). + BinaryPrimitives.WriteInt32LittleEndian(clear[(JetFormatBase.FreePagesMapPointerOffset - b)..], 0x00000100); // free map: page 1, row 0 + BinaryPrimitives.WriteInt32LittleEndian(clear[(JetFormatBase.ReleasedPagesMapPointerOffset - b)..], 0x00000101); // released map: page 1, row 1 // 0x20..0x2C: system-catalog bootstrap pointers = MSysObjects/ACEs/Queries/Relationships pages. BinaryPrimitives.WriteInt32LittleEndian(clear[(0x20 - b)..], 2); BinaryPrimitives.WriteInt32LittleEndian(clear[(0x24 - b)..], 3); @@ -252,6 +253,18 @@ private static readonly (string Name, ColumnSpec[] Columns)[] MSysComplexTypeTab public static void CreateEmpty(string path, byte version = 0x02, Collation? collation = null) { Collation sortOrder = collation ?? Collation.GeneralLegacy; + + // ACE 15 (0x04) can be read but never created: ACE REFUSES a file carrying the byte, an empty one + // included, and restamping 0x14 to 0x03 opens the identical bytes + // (Ace_refuses_the_0x04_version_byte_and_nothing_else_about_the_file). FromVersionByte still maps 0x04 + // onto the 0x03 layout, so reading one stays supported; only writing it is refused. + if (version == (byte)JetVersion.Version15_2013) + throw new NotSupportedException( + $"Cannot create a database at {nameof(JetVersion.Version15_2013)} (version byte 0x04): the " + + "Access engine refuses to open a file stamped with it. Access 2013 writes the Access 2010 " + + $"format, so use {nameof(JetVersion.Version14_2010)} for a 2013-era database. Existing 0x04 " + + "files can still be opened for reading."); + JetFormatBase format = JetFormatBase.FromVersionByte(version); // Jet 4 (0x01, the Access 2000 / 2002-2003 `.mdb`) and the ACCDB versions can both be created; the diff --git a/src/LibRed/LibRed.Core/Storage/IndexKeyEncoder.cs b/src/LibRed/LibRed.Core/Storage/IndexKeyEncoder.cs index a0cb14756..765bea224 100644 --- a/src/LibRed/LibRed.Core/Storage/IndexKeyEncoder.cs +++ b/src/LibRed/LibRed.Core/Storage/IndexKeyEncoder.cs @@ -1,6 +1,7 @@ using System.Buffers.Binary; using System.Globalization; using System.Runtime.InteropServices; +using EntityFrameworkCore.Jet.Data; using LibRed.Catalog; using LibRed.Formats; using LibRed.Storage.Types; @@ -280,7 +281,7 @@ private static byte[] EncodeFixed(ColumnDef column, object value) case JetDataType.Int32: return EncodeInteger(Convert.ToInt32(value, c), 4); case JetDataType.Currency: - return EncodeInteger((long)decimal.Round(Convert.ToDecimal(value, c) * 10000m), 8); + return EncodeInteger((long)decimal.Round(JetDecimalConverter.ToDecimal(value, c) * 10000m), 8); case JetDataType.Int64: // BIGINT — verified against ACE across 0, ±1, ±42 and both extremes return EncodeInteger(Convert.ToInt64(value, c), 8); case JetDataType.Single: @@ -290,7 +291,7 @@ private static byte[] EncodeFixed(ColumnDef column, object value) case JetDataType.DateTime: return EncodeFloatBits(BitConverter.DoubleToInt64Bits(Convert.ToDateTime(value, c).ToOADate()), 8); case JetDataType.FixedPoint: - return EncodeFixedPoint(Convert.ToDecimal(value, c), column.Scale); + return EncodeFixedPoint(JetDecimalConverter.ToDecimal(value, c), column.Scale); default: throw new NotSupportedException($"Index key type {column.Type} is not encodable."); } @@ -309,7 +310,9 @@ private static byte[] EncodeFixedPoint(decimal value, byte scale) { decimal factor = 1m; for (int i = 0; i < scale; i++) factor *= 10m; - decimal magnitude = decimal.Truncate(decimal.Round(Math.Abs(value) * factor, 0)); + // Truncated toward zero, matching ACE and — necessarily — JetTypeCodec.EncodeNumeric: quantise a key + // differently from its row and the value is indexed under a number the row does not contain. + decimal magnitude = decimal.Truncate(Math.Abs(value) * factor); int[] bits = decimal.GetBits(magnitude); // [lo, mid, hi, flags]; magnitude has scale 0 var key = new byte[17]; diff --git a/src/LibRed/LibRed.Core/Storage/JetTextComparer.cs b/src/LibRed/LibRed.Core/Storage/JetTextComparer.cs new file mode 100644 index 000000000..e01ef7036 --- /dev/null +++ b/src/LibRed/LibRed.Core/Storage/JetTextComparer.cs @@ -0,0 +1,22 @@ +using System.Runtime.InteropServices; + +namespace LibRed.Storage; + +/// +/// Text compared in Jet's General sort order — the order ACE's text index keys hold, and the one it compares text in +/// (case folded, accents significant, ß as ss, hyphens and apostrophes weighed only after the letters, +/// trailing spaces ignored). See . +/// +public static class JetTextComparer +{ + /// The sign of against , or null when either holds a character + /// the order does not cover. + public static int? Compare(string a, string b) + { + var left = new List(a.Length + 4); + var right = new List(b.Length + 4); + if (!JetTextCollation.TryEncode(a, left) || !JetTextCollation.TryEncode(b, right)) + return null; + return Math.Sign(CollectionsMarshal.AsSpan(left).SequenceCompareTo(CollectionsMarshal.AsSpan(right))); + } +} diff --git a/src/LibRed/LibRed.Core/Storage/LongValueReader.cs b/src/LibRed/LibRed.Core/Storage/LongValueReader.cs index a38602072..621f7c44f 100644 --- a/src/LibRed/LibRed.Core/Storage/LongValueReader.cs +++ b/src/LibRed/LibRed.Core/Storage/LongValueReader.cs @@ -10,7 +10,8 @@ namespace LibRed.Storage; /// /// /// Descriptor layout: bytes 0-3 = length with storage flags in the high two bits, bytes 4-7 = a -/// row+page pointer to the first LVAL chunk, bytes 8-11 reserved. Flags: +/// row+page pointer to the first LVAL chunk, bytes 8-11 the chain stamp (chained form only, checked against +/// the first chain page — see ). Flags: /// 0x80 = inline (payload follows the descriptor); 0x40 = single LVAL page (the row is /// the whole payload); otherwise the payload is chained across LVAL pages, each row /// beginning with a 4-byte pointer to the next chunk. @@ -54,9 +55,37 @@ internal byte[] ResolveWithPages(ReadOnlySpan descriptor, out IReadOnlyLis return value; } + VerifyChainStamp(descriptor, page); return ReadChain(page, row, length, out pages); } + /// + /// A chained descriptor and the first page of its chain carry the same 4-byte stamp, minted when + /// the chain was written. Disagreement means the page is no longer the one this descriptor was written + /// against — the chain was rewritten, or its pages were freed and reused under another value — so the + /// bytes behind the pointer belong to something else and must not be returned as this value. + /// + /// + /// ACE enforces this and reports it as "you and another user are attempting to change the same data at + /// the same time"; the diagnosis is the point, even if the wording is about the cause rather than what + /// was found. LibRed is single-writer, so it cannot produce the interleaving ACE guards against, but it + /// can be handed a file another engine wrote and must not read a stale chain as though it were live. + /// Only the entry page is checked, because every chunk after it is reached from a page already validated. + /// + private void VerifyChainStamp(ReadOnlySpan descriptor, int page) + { + if (page <= 0 || page >= _channel.PageCount) return; // ReadChain reports the bad pointer itself + + uint declared = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian( + descriptor.Slice(LongValueFormat.ChainStampOffset, 4)); + uint stored = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian( + _channel.ReadPageShared(page).Span.Slice(_channel.Format.DataChainStampOffset, 4)); + if (declared != stored) + throw new InvalidDataException( + $"Long-value chain at page {page} carries stamp 0x{stored:X8} but its descriptor declares " + + $"0x{declared:X8}; the chain is not the one this row was written against."); + } + private byte[] ReadChain(int page, int row, int length, out IReadOnlyList pages) { var result = new byte[length]; diff --git a/src/LibRed/LibRed.Core/Storage/LongValueWriter.cs b/src/LibRed/LibRed.Core/Storage/LongValueWriter.cs index 4724775bb..4ec7d192b 100644 --- a/src/LibRed/LibRed.Core/Storage/LongValueWriter.cs +++ b/src/LibRed/LibRed.Core/Storage/LongValueWriter.cs @@ -65,6 +65,14 @@ public LongValueResult Write(byte[] payload) var pages = new int[chunkCount]; for (int i = 0; i < chunkCount; i++) pages[i] = _allocator.Allocate(); + // The chain stamp goes in two places and must match in both: here on the first chunk page and in the + // descriptor below. It is a version tag on the CHAIN, minted per write of it, so a reader arriving + // through a descriptor can tell that the pages it is about to follow are the ones that descriptor was + // written against and not a later value's. Only the entry page carries it — every chunk after that is + // reached from a page already validated. ACE uses GetTickCount(); the value is arbitrary and only the + // agreement is checked, so matching its choice keeps our pages the shape Access produces. + uint stamp = (uint)Environment.TickCount; + for (int i = 0; i < chunkCount; i++) { int start = i * MaxChunkData; @@ -77,10 +85,11 @@ public LongValueResult Write(byte[] payload) row[2] = (byte)(nextPage >> 8); row[3] = (byte)(nextPage >> 16); payload.AsSpan(start, len).CopyTo(row.AsSpan(4)); - WriteChunkPage(pages[i], row); + WriteChunkPage(pages[i], row, i == 0 ? stamp : 0); } - return new LongValueResult(Descriptor(payload.Length, LongValueFormat.FlagChained, pages[0]), pages, FreePage: 0); + return new LongValueResult( + Descriptor(payload.Length, LongValueFormat.FlagChained, pages[0], stamp: stamp), pages, FreePage: 0); } /// Allocates a fresh LVAL page, writes as its row 0, and returns the @@ -148,14 +157,17 @@ public int WriteNewPage(byte[] row) public static byte[] SinglePageDescriptor(int length, int page, int row) => Descriptor(length, LongValueFormat.FlagSinglePage, page, row); - /// Writes one row () to a fresh LVAL data page, packed from the page end. - private void WriteChunkPage(int pageNumber, byte[] row) + /// Writes one row () to a fresh LVAL data page, packed from the page end. + /// is the chain stamp for the first page of a chain, and zero everywhere else — + /// which is what ACE writes on a single-page value and on every chunk after the first. + private void WriteChunkPage(int pageNumber, byte[] row, uint stamp = 0) { JetFormatBase format = _channel.Format; var page = new byte[format.PageSize]; page[0] = (byte)PageType.DataPage; page[1] = 0x01; // page flags (observed constant) BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(format.DataOwnerOffset, 4), LongValueFormat.LvalMarker); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(format.DataChainStampOffset, 4), stamp); int offset = format.PageSize - row.Length; row.CopyTo(page.AsSpan(offset)); @@ -166,8 +178,10 @@ private void WriteChunkPage(int pageNumber, byte[] row) _channel.WritePage(pageNumber, page); } - /// Builds the 12-byte descriptor: 4-byte length with storage flags, row/page, reserved. - private static byte[] Descriptor(int length, byte flag, int firstPage, int row = 0) + /// Builds the 12-byte descriptor: 4-byte length with storage flags, row/page, chain stamp. The + /// stamp is non-zero only on the chained form, where it repeats the first chain page's own — ACE leaves it + /// zero on the inline and single-page forms, and checks it on neither. + private static byte[] Descriptor(int length, byte flag, int firstPage, int row = 0, uint stamp = 0) { LongValueFormat.ValidateLength(length); var d = new byte[12]; @@ -176,6 +190,7 @@ private static byte[] Descriptor(int length, byte flag, int firstPage, int row = d[5] = (byte)firstPage; d[6] = (byte)(firstPage >> 8); d[7] = (byte)(firstPage >> 16); + BinaryPrimitives.WriteUInt32LittleEndian(d.AsSpan(LongValueFormat.ChainStampOffset, 4), stamp); return d; } } diff --git a/src/LibRed/LibRed.Core/Storage/PageAllocator.cs b/src/LibRed/LibRed.Core/Storage/PageAllocator.cs index 8512fcc37..a20f5a398 100644 --- a/src/LibRed/LibRed.Core/Storage/PageAllocator.cs +++ b/src/LibRed/LibRed.Core/Storage/PageAllocator.cs @@ -6,14 +6,21 @@ namespace LibRed.Storage; /// -/// Allocates database pages the way Access does: through the **global free-pages map** at page 1, -/// row 0 (an inline usage map where a set bit marks a free page). Allocation takes a free page, -/// clears its bit (so it is no longer free), and returns it — reusing freed pages rather than -/// always growing the file. The file is grown only when no free page is available. +/// Allocates database pages the way Access does: through the **global free-pages map**, the usage-map record +/// page 0 names at 0x18 (an inline or reference map where a set bit marks a free page). Allocation takes a +/// free page, clears its bit (so it is no longer free), and returns it — reusing freed pages rather than always +/// growing the file. The file is grown only when no free page is available. /// +/// +/// Pages set in the **global released-pages map** (named at 0x1C) are never allocated, as ACE never +/// allocates them: they were released by a session that has not yet merged them back into the free map. Both +/// maps are located only through their page-0 pointers, row included — page 1 rows 0 and 1 in every file ACE +/// writes, but ACE follows the pointers wherever they lead (docs/format/page-05-usage-maps.md §9.1). +/// Freed pages take one of two routes, as ACE's do: makes a page reusable at once, and +/// holds it until runs when the handle closes. +/// public sealed class PageAllocator(PageChannel channel) { - private const int GlobalMapPage = 1; private const byte InlineMapType = 0x00; private const byte ReferenceMapType = 0x01; /// Bytes preceding the bitmap on a dedicated usage-bitmap page (type 0x05). @@ -25,103 +32,330 @@ public sealed class PageAllocator(PageChannel channel) private readonly PageChannel _channel = channel; + /// One of the two global map records, as read through its page-0 pointer. + private sealed record MapRecord(string Name, int PageNumber, int Row, byte[] Page, RowSlot Slot) + { + public ReadOnlySpan Record => Page.AsSpan(Slot.Offset, Slot.Length); + public byte Type => Page[Slot.Offset]; + } + public int Allocate() { - (byte[] page, RowSlot slot) = ReadGlobalMap(); - int mapOffset = slot.Offset; - byte mapType = page[mapOffset]; - if (mapType == ReferenceMapType) - return AllocateFromReferenceMap(page.AsSpan(mapOffset, slot.Length)); - if (mapType != InlineMapType) - throw new InvalidDataException($"Global free-pages map has unknown type 0x{mapType:X2}."); - if (slot.Length < 5) - throw new InvalidDataException("Global inline free-pages map is shorter than its 5-byte header."); + (MapRecord free, MapRecord released) = ReadGlobalMaps(); + var releasedPages = new ReleasedPages(this, released); + if (free.Type == ReferenceMapType) + return AllocateFromReferenceMap(free, released, releasedPages); + byte[] page = free.Page; + int mapOffset = free.Slot.Offset; int startPage = BinaryPrimitives.ReadInt32LittleEndian(page.AsSpan(mapOffset + 1, 4)); int bitmapStart = mapOffset + 5; - int bitmapEnd = mapOffset + slot.Length; + int bitmapEnd = mapOffset + free.Slot.Length; for (int i = bitmapStart; i < bitmapEnd; i++) { - if (page[i] == 0) continue; - int bit = BitOperations.TrailingZeroCount(page[i]); - int allocated = startPage + (i - bitmapStart) * 8 + bit; - ValidateReusablePage(allocated, "inline free bit", allowAppendBoundary: true); - EnsurePhysicalAllocation(allocated); - page[i] &= (byte)~(1 << bit); // no longer free - _channel.WritePage(GlobalMapPage, page); - return allocated; + int bits = page[i]; + while (bits != 0) + { + int bit = BitOperations.TrailingZeroCount(bits); + bits &= bits - 1; + int allocated = startPage + (i - bitmapStart) * 8 + bit; + if (releasedPages.Contains(allocated)) continue; // released, not yet reusable + ValidateReusablePage(allocated, "inline free bit", free, released, AppendBoundary(releasedPages)); + EnsurePhysicalAllocation(allocated, releasedPages); + page[i] &= (byte)~(1 << bit); // no longer free + _channel.WritePage(free.PageNumber, page); + return allocated; + } } // An unrepresented page is not safely recorded as used. Grow the global map before appending. return GrowAndAllocate(); } - /// Returns a page to the global free-pages map (sets its bit) so it can be reused — the inverse - /// of . Used when dropping an index frees its B-tree pages, matching Access. + /// Returns a page to the global free-pages map now (sets its bit), so it can be reused — the + /// inverse of . ACE frees this way only the pages of a long value an UPDATE replaces; + /// every other freed page is held until the session closes, through . public void Free(int page) { - ValidateReusablePage(page, "page being freed", allowAppendBoundary: false); - (byte[] p, RowSlot slot) = ReadGlobalMap(); + (MapRecord free, MapRecord released) = ReadGlobalMaps(); + ValidateReusablePage(page, "page being freed", free, released, _channel.PageCount - 1); - int mapOffset = slot.Offset; - byte mapType = p[mapOffset]; - if (mapType == ReferenceMapType) + if (free.Type == ReferenceMapType) { - FreeInReferenceMap(p.AsSpan(mapOffset, slot.Length), page); + FreeInReferenceMap(free, released, page); return; } - if (mapType != InlineMapType) - throw new InvalidDataException($"Global free-pages map has unknown type 0x{mapType:X2}."); - if (slot.Length < 5) - throw new InvalidDataException("Global inline free-pages map is shorter than its 5-byte header."); + byte[] p = free.Page; + int mapOffset = free.Slot.Offset; int startPage = BinaryPrimitives.ReadInt32LittleEndian(p.AsSpan(mapOffset + 1, 4)); int bit = page - startPage; int byteIndex = mapOffset + 5 + bit / 8; - if (bit < 0 || byteIndex >= mapOffset + slot.Length) return; // outside the inline window + if (bit < 0 || byteIndex >= mapOffset + free.Slot.Length) return; // outside the inline window p[byteIndex] |= (byte)(1 << (bit % 8)); - _channel.WritePage(GlobalMapPage, p); + _channel.WritePage(free.PageNumber, p); + } + + /// + /// Frees a page the way ACE frees the pages of a deleted row's long values, a dropped index and a dropped + /// table: it is not reusable while this handle stays open, and goes back to the global free map only when + /// runs at close. Inside a transaction it is released only if the + /// transaction commits. + /// + public void Release(int page) + { + (MapRecord free, MapRecord released) = ReadGlobalMaps(); + ValidateReusablePage(page, "page being released", free, released, _channel.PageCount - 1); + _channel.ReleaseAtClose(page); + } + + /// + /// What ACE does at close: every page this handle released, and every page already set in the global + /// released-pages map, goes back to the global free-pages map, and the released map is cleared. First an + /// inline released map is sized to cover the highest page released (), which + /// can convert it to reference form. Writes nothing when nothing was released and this handle changed + /// nothing. + /// + public void ReturnReleasedPages() + { + SortedSet pages = [.. _channel.PagesReleasedAtClose]; + if (pages.Count == 0 && !_channel.HasPublishedWrites) return; + (_, MapRecord released) = ReadGlobalMaps(); + pages.UnionWith(ReleasedMapPages(released)); + if (pages.Count == 0) return; + + bool ownTransaction = !_channel.InTransaction; + if (ownTransaction) _channel.BeginTransaction(); + try + { + // Sized before the merge: a conversion allocates its bitmap pages from the free map as it stands, + // without the pages being released. + SizeReleasedMap(pages); + + // A released page past the end of the file was never materialized; the free map already records + // every page past the end as free. + foreach (int page in pages) + if (page < _channel.PageCount) Free(page); + + (_, released) = ReadGlobalMaps(); + ClearReleasedMap(released); + _channel.ClearPagesReleasedAtClose(); + if (ownTransaction) _channel.CommitTransaction(flush: false); + } + catch + { + if (ownTransaction) _channel.RollbackTransaction(); + throw; + } + } + + /// The pages set in the global released-pages map, inline or reference form. + private IEnumerable ReleasedMapPages(MapRecord released) + { + ReadOnlySpan record = released.Record; + var pages = new List(); + if (released.Type == InlineMapType) + { + int start = BinaryPrimitives.ReadInt32LittleEndian(record.Slice(1, 4)); + for (int i = 5; i < record.Length; i++) + for (int bits = record[i]; bits != 0; bits &= bits - 1) + pages.Add(start + (i - 5) * 8 + BitOperations.TrailingZeroCount(bits)); + return pages; + } + + int pagesPerBitmap = (_channel.PageSize - BitmapPageHeaderSize) * 8; + for (int slot = 0; slot < ReferenceMapSlots; slot++) + { + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(record.Slice(1 + slot * 4, 4)); + if (bitmapPage == 0) continue; + ReadOnlySpan bitmap = _channel.ReadPage(bitmapPage).Span; + for (int i = BitmapPageHeaderSize; i < bitmap.Length; i++) + for (int bits = bitmap[i]; bits != 0; bits &= bits - 1) + pages.Add(slot * pagesPerBitmap + (i - BitmapPageHeaderSize) * 8 + BitOperations.TrailingZeroCount(bits)); + } + return pages; + } + + /// + /// Sizes the released-pages map for the way ACE's close does, before they are merged. + /// An inline record that already covers them is left alone. Otherwise it is lengthened, keeping its start + /// page, to the shortest that covers the highest page — the 5-byte header, then the bitmap in whole 4-byte + /// words — as long as its holder keeps 4 bytes free. When that is too long, the window moves instead: the + /// start becomes the lowest page released rounded down to a byte, and the record is sized from there. When + /// even that is too long, the record is grown at its old start to cover the highest released page it can + /// reach, the released pages it covers are marked in it, and it is converted to reference form: a bitmap page is allocated for each range + /// holding a released page, in range order, and a 69-byte reference record takes its place, the longer + /// record's bytes staying on the page below it. A map already in reference form gains a bitmap page for each + /// range holding a released page that it has none for. + /// + private void SizeReleasedMap(SortedSet pages) + { + (_, MapRecord released) = ReadGlobalMaps(); + if (released.Type == ReferenceMapType) + { + byte[] existing = released.Record.ToArray(); + if (!AddReleasedBitmapPages(existing, pages)) return; + (_, released) = ReadGlobalMaps(); + LayMapRecord(released, existing); + return; + } + + int start = BinaryPrimitives.ReadInt32LittleEndian(released.Record.Slice(1, 4)); + int lowest = pages.Min, highest = pages.Max; + int Covering(int from) => 5 + ((highest - from) / 8 + 1 + 3) / 4 * 4; + if (lowest >= start && highest < start + (released.Slot.Length - 5) * 8) return; + + var holder = new DataPage(); + holder.Read(new PageBuffer(released.Page, released.PageNumber), _channel.Format); + int others = 0; + for (int row = 0; row < holder.RowCount; row++) + if (row != released.Row) others += holder.Rows[row].Length; + int room = _channel.PageSize - (_channel.Format.DataRowDirectoryOffset + holder.RowCount * 2) - others - 4; + int longest = Math.Max(released.Slot.Length, 5 + (room - 5) / 4 * 4); + + if (lowest >= start && Covering(start) <= longest) + { + var grown = new byte[Covering(start)]; + released.Record[..5].CopyTo(grown); + LayMapRecord(released, grown); + return; + } + + int moved = lowest / 8 * 8; + if (Covering(moved) <= longest) + { + var window = new byte[Math.Max(Covering(moved), released.Slot.Length)]; + BinaryPrimitives.WriteInt32LittleEndian(window.AsSpan(1, 4), moved); + LayMapRecord(released, window); + return; + } + + // Grown only as far as the highest released page it can still cover — the whole of its longest length only + // when released pages reach that far. + int reach = start + (longest - 5) * 8 - 1; + SortedSet reachable = pages.GetViewBetween(Math.Min(start, reach), reach); + int covered = reachable.Count == 0 ? released.Slot.Length + : Math.Max(released.Slot.Length, 5 + ((reachable.Max - start) / 8 + 1 + 3) / 4 * 4); + var record = new byte[covered]; + released.Record[..5].CopyTo(record); + foreach (int page in pages.GetViewBetween(start, start + (record.Length - 5) * 8 - 1)) + record[5 + (page - start) / 8] |= (byte)(1 << ((page - start) % 8)); + LayMapRecord(released, record); + + var reference = new byte[1 + ReferenceMapSlots * 4]; + reference[0] = ReferenceMapType; + AddReleasedBitmapPages(reference, pages); + (_, released) = ReadGlobalMaps(); + LayMapRecord(released, reference); + } + + /// Allocates an empty bitmap page, in range order, for every range of that + /// the reference record has no bitmap page for, and writes its pointer into the record. Returns whether any + /// was added. + private bool AddReleasedBitmapPages(byte[] reference, SortedSet pages) + { + int pagesPerBitmap = (_channel.PageSize - BitmapPageHeaderSize) * 8; + bool added = false; + foreach (int slot in pages.Select(p => p / pagesPerBitmap).Distinct()) + { + if (slot >= ReferenceMapSlots) + throw new InvalidDataException($"Released page {pages.Max} lies past the global map's bitmap slots."); + if (BinaryPrimitives.ReadInt32LittleEndian(reference.AsSpan(1 + slot * 4)) != 0) continue; + int bitmapPage = Allocate(); + var bitmap = new byte[_channel.PageSize]; + bitmap[0] = (byte)PageType.PageUsageBitmap; + bitmap[1] = 1; + _channel.WritePage(bitmapPage, bitmap); + BinaryPrimitives.WriteInt32LittleEndian(reference.AsSpan(1 + slot * 4), bitmapPage); + added = true; + } + return added; + } + + /// Replaces a global map record, repacking its holder's records from the page end, and lays the result + /// over the page as it stands: bytes a moved record vacates are not cleared, as ACE leaves them. + private void LayMapRecord(MapRecord map, byte[] record) + { + byte[] page = _channel.ReadPage(map.PageNumber).Span.ToArray(); + var holder = new DataPage(); + holder.Read(new PageBuffer(page, map.PageNumber), _channel.Format); + byte[] repacked = UsageMapWriter.ReplaceMapRecord(page, holder, _channel.Format, map.Row, record, out _) + ?? throw new InvalidDataException($"Global {map.Name} map cannot fit its holder page."); + + var laid = new DataPage(); + laid.Read(new PageBuffer(repacked, map.PageNumber), _channel.Format); + repacked.AsSpan(0, _channel.Format.DataRowDirectoryOffset + laid.RowCount * 2).CopyTo(page); + foreach (RowSlot slot in laid.Rows) + repacked.AsSpan(slot.Offset, slot.Length).CopyTo(page.AsSpan(slot.Offset)); + _channel.WritePage(map.PageNumber, page); + } + + /// Clears every bit of the global released-pages map: in place for an inline record, and on each + /// bitmap page, header kept, for a reference record. + private void ClearReleasedMap(MapRecord released) + { + if (released.Type == ReferenceMapType) + { + ReadOnlySpan map = released.Record; + for (int slot = 0; slot < ReferenceMapSlots; slot++) + { + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1 + slot * 4, 4)); + if (bitmapPage == 0) continue; + byte[] bitmap = _channel.ReadPage(bitmapPage).Span.ToArray(); + bitmap.AsSpan(BitmapPageHeaderSize).Clear(); + _channel.WritePage(bitmapPage, bitmap); + } + return; + } + + released.Page.AsSpan(released.Slot.Offset + 5, released.Slot.Length - 5).Clear(); + _channel.WritePage(released.PageNumber, released.Page); } + /// + /// Checks that page 0's two global map pointers name distinct, well-formed usage-map records inside the + /// file. A pointer past the end of the file makes ACE mark the database corrupt, and one naming anything + /// other than a usage map fails at the first allocation, so a writable open refuses both up front. + /// + public void ValidateGlobalMaps() => ReadGlobalMaps(); + /// Allocates from a reference-type global free map (huge databases): the record is a list of /// pointers to dedicated bitmap pages (type 0x05), pointer k covering the page range starting at /// k × (pageSize − 4) × 8. A **set bit is a free page** (the global map's sense, opposite of a /// per-table owned map). Finds the first free page, clears its bit on the bitmap page, and returns it; /// grows the file when no bitmap records a free page. - private int AllocateFromReferenceMap(ReadOnlySpan map) + private int AllocateFromReferenceMap(MapRecord free, MapRecord released, ReleasedPages releasedPages) { var format = _channel.Format; - ValidateReferenceMap(map); + ReadOnlySpan map = free.Record; int pagesPerBitmap = (format.PageSize - BitmapPageHeaderSize) * 8; - var bitmapPages = new HashSet(); - for (int slot = 0; slot < ReferenceMapSlots; slot++) - { - int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1 + slot * 4, 4)); - if (bitmapPage == 0) continue; - ValidateBitmapPage(bitmapPage); - if (!bitmapPages.Add(bitmapPage)) - throw new InvalidDataException($"Global reference free map repeats bitmap page {bitmapPage}."); - } + HashSet bitmapPages = ReferenceBitmapPages(free, released); for (int slot = 0; slot < ReferenceMapSlots; slot++) { int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1 + slot * 4, 4)); if (bitmapPage == 0) continue; // no bitmap page allocated for this range - byte[] bitmap = ValidateBitmapPage(bitmapPage); + byte[] bitmap = ValidateBitmapPage(bitmapPage, free, released); for (int i = BitmapPageHeaderSize; i < format.PageSize; i++) { - if (bitmap[i] == 0) continue; - int bit = BitOperations.TrailingZeroCount(bitmap[i]); - int allocated = slot * pagesPerBitmap + (i - BitmapPageHeaderSize) * 8 + bit; - ValidateReusablePage(allocated, $"reference-map slot {slot} free bit", allowAppendBoundary: true); - if (bitmapPages.Contains(allocated)) - throw new InvalidDataException($"Global free map marks bitmap page {allocated} itself as free."); - EnsurePhysicalAllocation(allocated); - bitmap[i] &= (byte)~(1 << bit); // no longer free - _channel.WritePage(bitmapPage, bitmap); - return allocated; + int bits = bitmap[i]; + while (bits != 0) + { + int bit = BitOperations.TrailingZeroCount(bits); + bits &= bits - 1; + int allocated = slot * pagesPerBitmap + (i - BitmapPageHeaderSize) * 8 + bit; + if (releasedPages.Contains(allocated)) continue; // released, not yet reusable + ValidateReusablePage(allocated, $"reference-map slot {slot} free bit", free, released, + AppendBoundary(releasedPages)); + if (bitmapPages.Contains(allocated)) + throw new InvalidDataException($"Global free map marks bitmap page {allocated} itself as free."); + EnsurePhysicalAllocation(allocated, releasedPages); + bitmap[i] &= (byte)~(1 << bit); // no longer free + _channel.WritePage(bitmapPage, bitmap); + return allocated; + } } } @@ -151,8 +385,9 @@ private int GrowAndAllocate() private int GrowAndAllocateCore() { - (byte[] page, RowSlot slot) = ReadGlobalMap(); - byte[] record = page.AsSpan(slot.Offset, slot.Length).ToArray(); + (MapRecord free, MapRecord released) = ReadGlobalMaps(); + var releasedPages = new ReleasedPages(this, released); + byte[] record = free.Record.ToArray(); int frontier = _channel.PageCount; if (record[0] == InlineMapType) { @@ -161,7 +396,7 @@ private int GrowAndAllocateCore() throw new NotSupportedException("Cannot grow a global inline map with a nonzero start page."); int bitmapBytes = ((frontier / 8 + 1 + 3) / 4) * 4; if (bitmapBytes <= record.Length - 5) - return _channel.AllocatePage(); // already represented as used + return AppendUnreleasedPage(releasedPages); // already represented as used var grown = new byte[5 + bitmapBytes]; // Preserve existing free bits; newly covered physical pages are already used. Only future @@ -170,12 +405,12 @@ private int GrowAndAllocateCore() for (int bit = frontier; bit < bitmapBytes * 8; bit++) grown[5 + bit / 8] |= (byte)(1 << (bit % 8)); var holder = new DataPage(); - holder.Read(_channel.ReadPage(GlobalMapPage), _channel.Format); - byte[]? rewritten = UsageMapWriter.ReplaceMapRecord(page, holder, _channel.Format, 0, grown, out _); + holder.Read(_channel.ReadPage(free.PageNumber), _channel.Format); + byte[]? rewritten = UsageMapWriter.ReplaceMapRecord(free.Page, holder, _channel.Format, free.Row, grown, out _); if (rewritten is not null && BinaryPrimitives.ReadUInt16LittleEndian(rewritten.AsSpan(_channel.Format.DataFreeSpaceOffset)) >= 4) { - _channel.WritePage(GlobalMapPage, rewritten); + _channel.WritePage(free.PageNumber, rewritten); return Allocate(); } @@ -188,7 +423,7 @@ private int GrowAndAllocateCore() { if (range >= ReferenceMapSlots) throw new NotSupportedException("Global allocation map has no remaining bitmap slots."); - int bitmap = _channel.AllocatePage(); + int bitmap = AppendUnreleasedPage(releasedPages); BinaryPrimitives.WriteInt32LittleEndian(record.AsSpan(1 + range * 4), bitmap); } for (int range = 0; range < ReferenceMapSlots; range++) @@ -196,25 +431,33 @@ private int GrowAndAllocateCore() int bitmap = BinaryPrimitives.ReadInt32LittleEndian(record.AsSpan(1 + range * 4)); if (bitmap != 0) WriteNewGlobalBitmap(bitmap, range); } - WriteGlobalRecord(record); + WriteGlobalRecord(free, record); return Allocate(); } - ValidateReferenceMap(record); int pagesPerBitmap = (_channel.PageSize - BitmapPageHeaderSize) * 8; int rangeIndex = frontier / pagesPerBitmap; if (rangeIndex >= ReferenceMapSlots) throw new NotSupportedException("Global allocation map has no remaining bitmap slots."); if (BinaryPrimitives.ReadInt32LittleEndian(record.AsSpan(1 + rangeIndex * 4)) != 0) - return _channel.AllocatePage(); // represented range, bit already clear + return AppendUnreleasedPage(releasedPages); // represented range, bit already clear - int newBitmap = _channel.AllocatePage(); + int newBitmap = AppendUnreleasedPage(releasedPages); BinaryPrimitives.WriteInt32LittleEndian(record.AsSpan(1 + rangeIndex * 4), newBitmap); WriteNewGlobalBitmap(newBitmap, rangeIndex); - WriteGlobalRecord(record); + WriteGlobalRecord(free, record); return Allocate(); } + /// Appends the next page past the end of the file that is not released. Released pages at the + /// frontier are materialized — the file stays contiguous — but not handed out. + private int AppendUnreleasedPage(ReleasedPages releasedPages) + { + while (releasedPages.Contains(_channel.PageCount)) + _channel.AllocatePage(); + return _channel.AllocatePage(); + } + private void WriteNewGlobalBitmap(int number, int range) { var bitmap = new byte[_channel.PageSize]; @@ -227,23 +470,23 @@ private void WriteNewGlobalBitmap(int number, int range) _channel.WritePage(number, bitmap); } - private void WriteGlobalRecord(byte[] record) + private void WriteGlobalRecord(MapRecord free, byte[] record) { - byte[] page = _channel.ReadPage(GlobalMapPage).Span.ToArray(); + byte[] page = _channel.ReadPage(free.PageNumber).Span.ToArray(); var holder = new DataPage(); - holder.Read(_channel.ReadPage(GlobalMapPage), _channel.Format); - byte[] rewritten = UsageMapWriter.ReplaceMapRecord(page, holder, _channel.Format, 0, record, out _) + holder.Read(_channel.ReadPage(free.PageNumber), _channel.Format); + byte[] rewritten = UsageMapWriter.ReplaceMapRecord(page, holder, _channel.Format, free.Row, record, out _) ?? throw new InvalidDataException("Global allocation map cannot fit its holder page."); - _channel.WritePage(GlobalMapPage, rewritten); + _channel.WritePage(free.PageNumber, rewritten); } /// Returns a page to a reference-type global free map by setting its bit on the bitmap page for /// its range. If that range has no bitmap page (e.g. a page grown past the map's coverage), the page is /// left unrecorded — it simply won't be reused, matching the pre-existing inline-window behaviour. - private void FreeInReferenceMap(ReadOnlySpan map, int page) + private void FreeInReferenceMap(MapRecord free, MapRecord released, int page) { var format = _channel.Format; - ValidateReferenceMap(map); + ReadOnlySpan map = free.Record; int pagesPerBitmap = (format.PageSize - BitmapPageHeaderSize) * 8; int slot = page / pagesPerBitmap; if (slot < 0 || slot >= ReferenceMapSlots) return; // beyond the map's ~2 GB reach @@ -252,58 +495,173 @@ private void FreeInReferenceMap(ReadOnlySpan map, int page) if (bitmapPage == 0) return; // range has no bitmap page — nothing to record into int bitInRange = page - slot * pagesPerBitmap; - byte[] bitmap = ValidateBitmapPage(bitmapPage); + byte[] bitmap = ValidateBitmapPage(bitmapPage, free, released); if (page == bitmapPage) throw new InvalidDataException($"Usage-map bitmap page {page} cannot be marked globally free."); bitmap[BitmapPageHeaderSize + bitInRange / 8] |= (byte)(1 << (bitInRange % 8)); _channel.WritePage(bitmapPage, bitmap); } - private (byte[] Page, RowSlot Slot) ReadGlobalMap() + /// Reads both global map records through page 0's pointers, and refuses pointers that are equal, + /// name a page outside the file, or name anything other than a usage-map record. + private (MapRecord Free, MapRecord Released) ReadGlobalMaps() { - if (_channel.PageCount <= GlobalMapPage) - throw new InvalidDataException("Database has no global free-pages map at page 1."); - PageBuffer buffer = _channel.ReadPage(GlobalMapPage); + ReadOnlySpan page0 = _channel.ReadPage(0).Span; + (int Row, int Page) freePointer = + DatabaseDefinitionPage.ReadMapPointer(page0, Formats.JetFormatBase.FreePagesMapPointerOffset); + (int Row, int Page) releasedPointer = + DatabaseDefinitionPage.ReadMapPointer(page0, Formats.JetFormatBase.ReleasedPagesMapPointerOffset); + if (freePointer == releasedPointer) + throw new InvalidDataException( + $"Page 0 names the same record (page {freePointer.Page}, row {freePointer.Row}) for the global " + + "free-pages and released-pages maps; they must be distinct."); + + MapRecord free = ReadMapRecord("free-pages", freePointer); + MapRecord released = ReadMapRecord("released-pages", releasedPointer); + ReferenceBitmapPages(free, released); + return (free, released); + } + + private MapRecord ReadMapRecord(string name, (int Row, int Page) pointer) + { + if (pointer.Page <= 0 || pointer.Page >= _channel.PageCount) + throw new InvalidDataException( + $"Page 0's global {name} map pointer names page {pointer.Page}, outside the file's pages 1..{_channel.PageCount - 1}."); + PageBuffer buffer = _channel.ReadPage(pointer.Page); + if (buffer.Span[0] != (byte)PageType.DataPage) + throw new InvalidDataException( + $"Page 0's global {name} map pointer names page {pointer.Page}, which is not a data page."); var data = new DataPage(); data.Read(buffer, _channel.Format); - if (data.RowCount < 1) - throw new InvalidDataException("Global free-pages map page has no row 0."); - RowSlot slot = data.Rows[0]; + if (pointer.Row >= data.RowCount) + throw new InvalidDataException( + $"Page 0's global {name} map pointer names row {pointer.Row} of page {pointer.Page}, which has {data.RowCount} rows."); + RowSlot slot = data.Rows[pointer.Row]; if (slot.IsDeleted || slot.HasOverflow || slot.Length == 0) - throw new InvalidDataException("Global free-pages map row 0 is deleted, overflowed, or empty."); - return (buffer.Span.ToArray(), slot); + throw new InvalidDataException( + $"Global {name} map (page {pointer.Page}, row {pointer.Row}) is deleted, overflowed, or empty."); + + var record = new MapRecord(name, pointer.Page, pointer.Row, buffer.Span.ToArray(), slot); + if (record.Type == InlineMapType) + { + if (slot.Length < 5) + throw new InvalidDataException($"Global inline {name} map is shorter than its 5-byte header."); + } + else if (record.Type == ReferenceMapType) + { + if (slot.Length != 1 + ReferenceMapSlots * 4) + throw new InvalidDataException( + $"Global reference {name} map must be exactly {1 + ReferenceMapSlots * 4} bytes; got {slot.Length}."); + } + else + { + throw new InvalidDataException($"Global {name} map has unknown type 0x{record.Type:X2}."); + } + return record; } - private static void ValidateReferenceMap(ReadOnlySpan map) + /// The bitmap pages the two reference-form maps own, each validated; a page may belong to only + /// one slot of one map. + private HashSet ReferenceBitmapPages(MapRecord free, MapRecord released) { - if (map.Length != 1 + ReferenceMapSlots * 4) - throw new InvalidDataException( - $"Global reference free map must be exactly {1 + ReferenceMapSlots * 4} bytes; got {map.Length}."); + var pages = new HashSet(); + foreach (MapRecord map in new[] { free, released }) + { + if (map.Type != ReferenceMapType) continue; + ReadOnlySpan record = map.Record; + for (int slot = 0; slot < ReferenceMapSlots; slot++) + { + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(record.Slice(1 + slot * 4, 4)); + if (bitmapPage == 0) continue; + ValidateBitmapPage(bitmapPage, free, released); + if (!pages.Add(bitmapPage)) + throw new InvalidDataException($"Global reference maps repeat bitmap page {bitmapPage}."); + } + } + return pages; } - private byte[] ValidateBitmapPage(int pageNumber) + private byte[] ValidateBitmapPage(int pageNumber, MapRecord free, MapRecord released) { - ValidateReusablePage(pageNumber, "usage-map bitmap pointer", allowAppendBoundary: false); + ValidateReusablePage(pageNumber, "usage-map bitmap pointer", free, released, _channel.PageCount - 1); byte[] page = _channel.ReadPage(pageNumber).Span.ToArray(); if (page[0] != (byte)PageType.PageUsageBitmap || page[1] != 0x01 || page[2] != 0 || page[3] != 0) - throw new InvalidDataException($"Global free-map pointer {pageNumber} does not target a valid bitmap page."); + throw new InvalidDataException($"Global map pointer {pageNumber} does not target a valid bitmap page."); return page; } - private void ValidateReusablePage(int page, string source, bool allowAppendBoundary) + /// Page 0 and the pages holding the two global maps are never allocatable; nor is a page past + /// . + private static void ValidateReusablePage(int page, string source, MapRecord free, MapRecord released, int maximum) { - int maximum = allowAppendBoundary ? _channel.PageCount : _channel.PageCount - 1; - if (page <= GlobalMapPage || page > maximum) + if (page <= 0 || page == free.PageNumber || page == released.PageNumber || page > maximum) throw new InvalidDataException( - $"Global {source} names page {page}, outside the reusable contiguous range 2..{maximum}."); + $"Global {source} names page {page}, which is page 0, a global-map holder page " + + $"({free.PageNumber}/{released.PageNumber}), or past the contiguous range ending at {maximum}."); + } + + /// The highest page an allocation may name: the next page past the end of the file, pushed + /// further by any released pages sitting at the frontier (they are materialized, not handed out). + private int AppendBoundary(ReleasedPages releasedPages) + { + int boundary = _channel.PageCount; + while (releasedPages.Contains(boundary)) boundary++; + return boundary; } - private void EnsurePhysicalAllocation(int page) + private void EnsurePhysicalAllocation(int page, ReleasedPages releasedPages) { + while (_channel.PageCount < page) + { + if (!releasedPages.Contains(_channel.PageCount)) + throw new InvalidDataException( + $"Global free map selected page {page} past a gap at page {_channel.PageCount} that is neither free nor released."); + _channel.AllocatePage(); + } if (page < _channel.PageCount) return; int allocated = _channel.AllocatePage(); if (allocated != page) throw new InvalidDataException( $"Global free map selected append page {page}, but contiguous allocation produced page {allocated}."); } + + /// Membership in the global released-pages map, inline or reference form. + private sealed class ReleasedPages + { + private readonly PageAllocator _owner; + private readonly MapRecord _map; + private readonly Dictionary _bitmaps = []; + + public ReleasedPages(PageAllocator owner, MapRecord map) + { + _owner = owner; + _map = map; + } + + public bool Contains(int page) + { + if (page < 0) return false; + ReadOnlySpan record = _map.Record; + if (_map.Type == InlineMapType) + { + int start = BinaryPrimitives.ReadInt32LittleEndian(record.Slice(1, 4)); + long bit = (long)page - start; + if (bit < 0 || bit / 8 >= record.Length - 5) return false; + return (record[5 + (int)(bit / 8)] & (1 << (int)(bit % 8))) != 0; + } + + int pagesPerBitmap = (_owner._channel.PageSize - BitmapPageHeaderSize) * 8; + int slot = page / pagesPerBitmap; + if (slot >= ReferenceMapSlots) return false; + if (!_bitmaps.TryGetValue(slot, out byte[]? bitmap)) + { + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(record.Slice(1 + slot * 4, 4)); + bitmap = bitmapPage == 0 ? null : _owner._channel.ReadPage(bitmapPage).Span.ToArray(); + _bitmaps[slot] = bitmap; + } + if (bitmap is null) return false; + int inRange = page - slot * pagesPerBitmap; + return (bitmap[BitmapPageHeaderSize + inRange / 8] & (1 << (inRange % 8))) != 0; + } + } } diff --git a/src/LibRed/LibRed.Core/Storage/RowEncoder.cs b/src/LibRed/LibRed.Core/Storage/RowEncoder.cs index 2a18d1744..d3070b3f8 100644 --- a/src/LibRed/LibRed.Core/Storage/RowEncoder.cs +++ b/src/LibRed/LibRed.Core/Storage/RowEncoder.cs @@ -50,7 +50,11 @@ public sealed class RowEncoder(IReadOnlyList columns, JetFormatBase f /// the cached value exactly as it was: ACE recomputes only when a referenced column is written, so /// recomputing unconditionally would write bytes ACE would not have (§3.4a). Null on INSERT, where every /// calculated column is computed fresh. - public byte[] Encode(object?[] values, IReadOnlyDictionary? preservedCalculated) + /// The row as its columns hold it, for the calculated columns to read, when + /// already carries long values as their on-disk descriptors. A memo's text is + /// what an expression reads, not the descriptor that points at it. Null when the two are the same. + public byte[] Encode(object?[] values, IReadOnlyDictionary? preservedCalculated, + object?[]? logicalValues = null) { if (values.Length != _columns.Count) throw new ArgumentException($"Expected {_columns.Count} values, got {values.Length}.", nameof(values)); @@ -90,7 +94,7 @@ public byte[] Encode(object?[] values, IReadOnlyDictionary? preserv { if (column.IsCalculated) { - varChunks[column.VariableIndex] = EncodeCalculated(column, values, preservedCalculated); + varChunks[column.VariableIndex] = EncodeCalculated(column, logicalValues ?? values, preservedCalculated); continue; } object? v = values[column.Index]; @@ -145,6 +149,25 @@ private static void EnsureFitsDeclaredLength(ColumnDef column, byte[] encoded) + $"{actual} {(text ? "characters" : "bytes")} into a column declared to hold {declared}."); } + /// + /// Smallest fixed region ACE writes in a row that has no variable trailer. It is a floor, not an + /// alignment: measured against ACE, a region of 0 bytes (a table of only Booleans, which occupy none) + /// is padded to 2 and 1 byte (a lone BYTE column) to 2, while 3 stays 3 — a three-BYTE table's + /// row is 6 bytes, odd region and all. A row that carries a variable trailer is exempt: ACE leaves a + /// TEXT-only table's fixed region at 0. + /// + /// + /// Matching it is not cosmetic. Without the pad an all-Boolean table of eight columns or fewer encodes to a + /// 3-byte record, and ACE misreads that record — every Boolean in it comes back False, whichever + /// engine created the table. Measured both ways round: ACE's own table filled by LibRed read False, and + /// LibRed's table filled by ACE read True, which is what pins the fault to the record rather than the TDEF. + /// The cliff is at 4 bytes — a 16-Boolean row (2-byte bitmap, so 4 bytes) reads back correctly — but ACE's + /// own writer never emits a record under 5, so the short form is simply a shape its reader has never met. + /// The TDEF's fixed-row length keeps the true, unpadded value: ACE stores 1 for a BYTE table while + /// writing 5-byte rows into it, so this rounding happens at row-write time and nowhere else. + /// + private const int MinFixedRegion = 2; + /// Assembles the on-disk row bytes from a prepared fixed region and the ordered variable chunks: /// [count][fixed][var data][var-offset table][numVar] (the variable section is omitted entirely when /// there are none) then [null bitmap]. The count and bitmap width are maxColumnId + 1; a @@ -175,14 +198,18 @@ internal static byte[] AssembleRow(int maxColumnId, ReadOnlySpan fixedRegi for (int j = 0; j < numVar; j++) varDataLength += varChunks[j].Length; int varSectionLen = numVar > 0 ? varDataLength + (numVar + 1) * 2 + 2 : 0; - var row = new byte[countSize + fixedRegion.Length + varSectionLen + nullBitmapSize]; + // ACE pads an all-fixed row's fixed region out to MinFixedRegion; a row with a variable trailer is + // left alone. The pad is zero bytes between the fixed values and the null bitmap. + int fixedLen = numVar > 0 ? fixedRegion.Length : Math.Max(fixedRegion.Length, MinFixedRegion); + + var row = new byte[countSize + fixedLen + varSectionLen + nullBitmapSize]; BinaryPrimitives.WriteUInt16LittleEndian(row, (ushort)count); fixedRegion.CopyTo(row.AsSpan(countSize)); int bitmapPos; if (numVar > 0) { - int varDataStart = countSize + fixedRegion.Length; + int varDataStart = countSize + fixedLen; int pos = varDataStart; for (int j = 0; j < numVar; j++) { varChunks[j].CopyTo(row.AsSpan(pos)); pos += varChunks[j].Length; } @@ -199,7 +226,7 @@ internal static byte[] AssembleRow(int maxColumnId, ReadOnlySpan fixedRegi BinaryPrimitives.WriteUInt16LittleEndian(row.AsSpan(numVarPos, 2), (ushort)numVar); bitmapPos = numVarPos + 2; } - else bitmapPos = countSize + fixedRegion.Length; + else bitmapPos = countSize + fixedLen; var liveIds = new HashSet(); foreach (ColumnDef column in columns) diff --git a/src/LibRed/LibRed.Core/Storage/RowInserter.cs b/src/LibRed/LibRed.Core/Storage/RowInserter.cs index df9951384..ad716d23a 100644 --- a/src/LibRed/LibRed.Core/Storage/RowInserter.cs +++ b/src/LibRed/LibRed.Core/Storage/RowInserter.cs @@ -19,6 +19,8 @@ public sealed class RowInserter(PageChannel channel, TableDef table) private readonly UsageMapWriter _usageMaps = new(channel); private readonly TableDef _table = table; + private bool HasCalculatedColumns => _table.Columns.Any(c => c.IsCalculated); + /// Encodes and writes (aligned to column Index) into the table. public void Insert(object?[] values) => Insert(values, updateIndexes: true); @@ -43,14 +45,16 @@ public void Insert(object?[] values, bool updateIndexes) // Index keys are encoded from the *logical* values. MaterializeLongValues replaces a memo/OLE value // with its on-disk LongValueDescriptor, and a Memo column IS indexable (its key is the collation key // of the first 255 characters), so snapshot the values first and key the index off that snapshot. - object?[] keyValues = updateIndexes ? (object?[])values.Clone() : values; + // A calculated column reads the same logical values, so the snapshot serves it too. + bool calculated = HasCalculatedColumns; + object?[] keyValues = updateIndexes || calculated ? (object?[])values.Clone() : values; MaterializeLongValues(values); // Encode first: the fixed-region length is pinned by any existing row (to match Access), // or derived from the columns for a just-created empty table. var encoder = new RowEncoder(_table.Columns, format, InferFixedDataLength(format), _table.VariableColumnCount, SpillCalculated); - byte[] record = encoder.Encode(values); + byte[] record = encoder.Encode(values, null, calculated ? keyValues : null); EnsureRecordFits(format, record); @@ -75,7 +79,9 @@ public void Insert(object?[] values, bool updateIndexes) _channel.WritePage(pageNumber, page); - UpdateTdefCounters(format, values, generatedAutoNumbers); + // Asked before the row's own entries go in, so an index "already has" a key only through another row. + HashSet newKeys = updateIndexes ? IndexesGainingANewKey(keyValues) : []; + UpdateTdefCounters(format, values, generatedAutoNumbers, newKeys); if (updateIndexes) UpdateIndexes(keyValues, new RowId(pageNumber, rowCount)); } @@ -93,6 +99,10 @@ public void Update(RowId id, object?[] values, IReadOnlySet changedColumns) JetFormatBase format = _channel.Format; RejectExplicitCalculatedValues(values, changedColumns); + // The row as its columns hold it, for a recomputed calculated column to read: the loop below swaps an + // unchanged memo's text for its on-disk descriptor, and MaterializeLongValues the changed ones. + object?[]? logicalValues = HasCalculatedColumns ? (object?[])values.Clone() : null; + // Long-value (memo/OLE) columns: keep an unchanged column's on-disk descriptor verbatim (so it is not // needlessly re-materialised onto fresh LVAL pages), and free a changed column's old chained pages. byte[] oldRow = ReadRowBytes(id); @@ -103,7 +113,7 @@ public void Update(RowId id, object?[] values, IReadOnlySet changedColumns) { if (column.Type is not (JetDataType.Memo or JetDataType.Ole)) continue; if (!oldDescriptors.TryGetValue(column.Index, out byte[]? oldDescriptor)) continue; // old value was null - if (changedColumns.Contains(column.Index)) FreeLongValue(column, oldDescriptor); + if (changedColumns.Contains(column.Index)) FreeLongValue(column, oldDescriptor, releaseAtClose: false); else values[column.Index] = new LongValueDescriptor(oldDescriptor); } @@ -131,7 +141,7 @@ public void Update(RowId id, object?[] values, IReadOnlySet changedColumns) if (preservedCalculated?.ContainsKey(column.Index) == true) continue; if (oldCalculated.TryGetValue(column.Index, out byte[]? stale) && stale.Length >= LongValueFormat.DescriptorSize) - FreeLongValue(column, stale); + FreeLongValue(column, stale, releaseAtClose: false); } MaterializeLongValues(values); @@ -142,7 +152,7 @@ public void Update(RowId id, object?[] values, IReadOnlySet changedColumns) // Order Details), which without the guard would overflow `new byte[len]`. var encoder = new RowEncoder(_table.Columns, format, InferFixedDataLength(format), _table.VariableColumnCount, SpillCalculated); - byte[] record = encoder.Encode(values, preservedCalculated); + byte[] record = encoder.Encode(values, preservedCalculated, logicalValues); // Here as well as on the insert path, and before the in-place rewrite rather than beside the // page-search: a row that grows past the cap but still fits its current page is rewritten where it @@ -254,12 +264,12 @@ public void Delete(RowId id) { JetFormatBase format = _channel.Format; - // Free the deleted row's chained long-value pages. + // Free the deleted row's chained long-value pages — held until close, as ACE holds them. var oldDescriptors = new RowDecoder(_table.Columns, format).LongValueRaw(ReadRowBytes(id)); foreach (ColumnDef column in _table.Columns) if ((column.Type is JetDataType.Memo or JetDataType.Ole || column.HasLongValueMap) && oldDescriptors.TryGetValue(column.Index, out byte[]? d)) - FreeLongValue(column, d); + FreeLongValue(column, d, releaseAtClose: true); byte[] page = ArrayPool.Shared.Rent(format.PageSize); try @@ -350,7 +360,7 @@ private void ReclaimRelocationTarget(JetFormatBase format, byte[] sourcePage, in /// 0FED 0FDA CFDA, with free space rising by 19 in each case. /// /// - private static void ReclaimRow(JetFormatBase format, byte[] page, int row) + internal static void ReclaimRow(JetFormatBase format, byte[] page, int row) { int rowCount = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(format.DataRowCountOffset, 2)); int Offset(int i) => BinaryPrimitives.ReadUInt16LittleEndian( @@ -438,7 +448,12 @@ private byte[] ReadRowBytes(RowId id) /// Inline (0x80) values have no pages; single-page (0x40) values share a page with others, so reclaiming /// their row is deferred (they are left in place — a small, shared-page leak). /// - private void FreeLongValue(ColumnDef column, byte[] descriptor) + /// + /// Measured against ACE on one connection: the pages of a value an UPDATE replaces are set in the global + /// free map at once, while a DELETE's are not reusable until the connection closes + /// (, see ). + /// + private void FreeLongValue(ColumnDef column, byte[] descriptor, bool releaseAtClose) { // The descriptor comes off the row's variable chunk, so its width is whatever the offset table said. // LongValueReader requires the full 12 bytes before reading any field; reclaiming has to agree, or a @@ -452,13 +467,21 @@ private void FreeLongValue(ColumnDef column, byte[] descriptor) byte flags = (byte)(descriptor[3] & LongValueFormat.FlagMask); if (flags is not (LongValueFormat.FlagInline or LongValueFormat.FlagSinglePage or LongValueFormat.FlagChained)) throw new InvalidDataException($"Long-value descriptor has unknown flags 0x{flags:X2}."); - // Inline (0x80) has no pages; single-page (0x40) shares its page with other values — neither is - // reclaimed here. Only a chained value owns pages outright. - if (flags != LongValueFormat.FlagChained) return; + // Inline (0x80) keeps its payload in the row, so there is nothing to give back. + if (flags == LongValueFormat.FlagInline) return; TableDefinitionPage definition = ReadDefinition(); definition.LongValueOwnedMaps.TryGetValue(column.ColumnId, out (int Row, int Page) owned); definition.LongValueFreeMaps.TryGetValue(column.ColumnId, out (int Row, int Page) free); + + // A single-page (0x40) value shares its page with other values, so the page goes back only once the + // last of them is gone — until then just its own row is retired. + if (flags == LongValueFormat.FlagSinglePage) + { + ReleasePackedValue(column, descriptor[5] | (descriptor[6] << 8) | (descriptor[7] << 16), + descriptor[4], owned, free, releaseAtClose); + return; + } var allocator = new PageAllocator(_channel); var reader = new LongValueReader(_channel); _ = reader.ResolveWithPages(descriptor, out IReadOnlyList pages); @@ -474,23 +497,86 @@ private void FreeLongValue(ColumnDef column, byte[] descriptor) // one per statement); a direct Core caller without one gets no more than any other multi-page write. foreach (int page in pages) { - allocator.Free(page); + if (releaseAtClose) allocator.Release(page); + else allocator.Free(page); _usageMaps.SetBit(owned.Row, owned.Page, page, set: false); _usageMaps.SetBit(free.Row, free.Page, page, set: false); } } - /// The raw bytes of slot on a data page (walks the packed rows). - private static byte[] SlotBytes(byte[] page, JetFormatBase format, int slot) + /// + /// Retires one value from a shared (single-page form) long-value page: its row becomes a 0-length + /// deleted+overflow tombstone and the page is re-laid, the surviving records packing from the page end + /// in slot order so the freed space is reclaimed. When nothing live is left the page is given back — its + /// type byte set to , its bit cleared from the column's owned + /// and free maps, and the page returned to the global allocator. + /// + /// + /// Measured against ACE, deleting 4 of 12 rows whose 400-character memos shared one page, then all 12: + /// + /// start 0x01 n=5 free=72 [3296,2496,1696,896,96] + /// 4 gone 0x01 n=5 free=3272 [4096DO,4096DO,4096DO,4096DO,3296] the survivor slid to the top + /// all 0x09 n=5 free=4072 [4096DO x5] + /// + /// This is where page type 0x09 comes from — an emptied packed long-value page, which the spec had + /// recorded as a released page of unidentified origin (page-09). Chained values are unaffected: they own + /// their pages outright and are freed below, leaving them at 0x01, which is why no experiment that + /// used a memo large enough to chain ever produced one. + /// + private void ReleasePackedValue(ColumnDef column, int pageNumber, int row, + (int Row, int Page) owned, (int Row, int Page) free, bool releaseAtClose) { - int prevEnd = format.PageSize; - for (int i = 0; i <= slot; i++) + JetFormatBase format = _channel.Format; + if (pageNumber <= 0 || pageNumber >= _channel.PageCount) + throw new InvalidDataException($"Long-value page pointer {pageNumber} is outside the file."); + + byte[] page = _channel.ReadPage(pageNumber).Span.ToArray(); + var holder = new DataPage(); + holder.Read(new PageBuffer(page, pageNumber), format); + if (!holder.IsLongValuePage) + throw new InvalidDataException($"Long-value pointer {pageNumber}:{row} targets a non-LVAL data page."); + if (row < 0 || row >= holder.RowCount) + throw new InvalidDataException( + $"Long-value row pointer {pageNumber}:{row} is outside the page's 0..{holder.RowCount - 1} range."); + if (holder.Rows[row].IsDeleted) return; // already retired; freeing twice must not double-count + + int dir = format.DataRowDirectoryOffset; + var records = new byte[holder.RowCount][]; + var flags = new ushort[holder.RowCount]; + for (int i = 0; i < holder.RowCount; i++) { - int offset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(format.DataRowDirectoryOffset + i * 2, 2)) & RowPointer.OffsetMask; - if (i == slot) return page.AsSpan(offset, prevEnd - offset).ToArray(); - prevEnd = offset; + bool dead = i == row || holder.Rows[i].IsDeleted; + records[i] = dead ? [] : page.AsSpan(holder.Rows[i].Offset, holder.Rows[i].Length).ToArray(); + flags[i] = (ushort)((dead ? RowPointer.DeletedFlag : 0) + | (dead || holder.Rows[i].HasOverflow ? RowPointer.OverflowFlag : 0)); } - throw new ArgumentOutOfRangeException(nameof(slot)); + + int offset = format.PageSize; + for (int i = 0; i < holder.RowCount; i++) + { + offset -= records[i].Length; // a 0-length tombstone lands on the page end, as ACE writes it + records[i].CopyTo(page.AsSpan(offset)); + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(dir + i * 2, 2), + (ushort)(flags[i] | (offset & RowPointer.OffsetMask))); + } + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(format.DataFreeSpaceOffset, 2), + (ushort)(offset - (dir + holder.RowCount * 2))); + + bool emptied = records.All(r => r.Length == 0); + if (emptied) page[0] = (byte)PageType.ReleasedLongValuePage; + _channel.WritePage(pageNumber, page); + + // A page that survives has room again, so it goes back into the column's free-pages map — the same + // map TryAppend consults when looking for somewhere to pack the next small value. + if (!emptied) + { + _usageMaps.SetBit(free.Row, free.Page, pageNumber, set: true); + return; + } + _usageMaps.SetBit(owned.Row, owned.Page, pageNumber, set: false); + _usageMaps.SetBit(free.Row, free.Page, pageNumber, set: false); + if (releaseAtClose) new PageAllocator(_channel).Release(pageNumber); + else new PageAllocator(_channel).Free(pageNumber); } /// Rejects the insert if a UNIQUE or PRIMARY index would gain a duplicate key. A row with a @@ -530,6 +616,27 @@ private void UpdateIndexes(object?[] values, RowId rowId) } } + /// + /// The root pages of the non-unique indexes the row brings a key they do not hold yet — the ones whose + /// unique-entry count advances (verified vs ACE: a key equal to one already in the index, collation included, + /// adds nothing; a key whose last row was deleted counts again; a Null key counts like any other, except in an + /// IGNORE NULL index, which does not hold it). A unique index always gains a new key, so it is not asked. + /// + private HashSet IndexesGainingANewKey(object?[] values) + { + var gaining = new HashSet(); + IndexWriter? writer = null; + foreach (IndexDef index in _table.Indexes + .Where(i => !i.IsUnique && i.RootPage > 0) + .GroupBy(i => i.RootPage).Select(g => g.First())) + { + if (index.IgnoreNulls && HasNullKey(index, values)) continue; + writer ??= new IndexWriter(_channel, _table); + if (!writer.KeyExists(index, values)) gaining.Add(index.RootPage); + } + return gaining; + } + private static bool HasNullKey(IndexDef index, object?[] values) => index.Columns.Any(c => values[c.Column.Index] is null or DBNull); @@ -861,8 +968,7 @@ private IReadOnlyList MapPages(int mapRow, int mapPage) private void AppendMapBits(List result, ReadOnlySpan bitmap, int startPage) { - // Bound read once, for the reason UsageMap.AppendSetBits records: outside a transaction PageCount is a - // file-length syscall, and a per-bit test costs real time on a hot path. Nothing here writes. + // Bound read once, as UsageMap.AppendSetBits does: the loop runs per bit on a hot path. Nothing here writes. int pageCount = _channel.PageCount; for (int i = 0; i < bitmap.Length; i++) @@ -943,14 +1049,15 @@ private static int RandomAutoNumber() /// to pick the *next* id = this + increment; leaving it stale makes Access reissue an existing id and /// reject the insert as a duplicate primary key.) /// Per-index **unique-entry count** (`0x3F + ordinal×12`, `+4`) — incremented by one for - /// each **unique** index (a unique index gets a distinct key per row). This is the cumulative - /// count Access advances on every insert and never decrements. The sibling **total-entry count** - /// (`+0`) is deliberately left untouched: Access does **not** maintain it live — it stays `0` - /// through inserts and is only written (to the row count) on compact/repair (verified: a live - /// ACE-inserted table reads total `0` while saved Northwind tables read total = row count). + /// each **unique** index (a unique index gets a distinct key per row) and each non-unique index in + /// (the row brings a key it does not hold yet). Access advances it on insert only, + /// once per real index, and never decrements it. The sibling **total-entry count** + /// (`+0`) is deliberately left untouched: Access does **not** maintain it live — it is written only + /// when the index is built (to its entries, in TableCreator's back-fill) or the file compacted + /// (verified: an index created on an empty table reads total `0` through any number of inserts). /// /// - private void UpdateTdefCounters(JetFormatBase format, object?[] values, bool[]? generatedAutoNumbers) + private void UpdateTdefCounters(JetFormatBase format, object?[] values, bool[]? generatedAutoNumbers, IReadOnlySet newKeys) { byte[] tdef = _channel.ReadPageShared(_table.DefinitionPage).Span.ToArray(); @@ -994,14 +1101,10 @@ private void UpdateTdefCounters(JetFormatBase format, object?[] values, bool[]? column.Seed = unchecked(newHighWater + column.Increment); } - // TODO(non-unique-index-stats): a non-unique index's unique-entry count must advance only - // when the inserted key is genuinely new (Access's cumulative-distinct semantics), which - // needs a probe of the existing keys. Only unique indexes advance it today — and LibRed does - // create non-unique ones (every FK backing index is one, and CREATE INDEX without UNIQUE), - // so this gap applies to the majority of indexes written, not to none of them. - foreach (IndexDef index in _table.Indexes) + // One count per real index, so a relationship's logical index sharing a real one does not advance it twice. + foreach (IndexDef index in _table.Indexes.GroupBy(i => i.RealIndexOrdinal).Select(g => g.First())) { - if (!index.IsUnique) continue; + if (!index.IsUnique && !newKeys.Contains(index.RootPage)) continue; if (index.IgnoreNulls && HasNullKey(index, values)) continue; // row was excluded from the index int statsUnique = format.TdefRealIndexBlockOffset + index.RealIndexOrdinal * format.RealIndexEntrySize + 4; int unique = BinaryPrimitives.ReadInt32LittleEndian(tdef.AsSpan(statsUnique, 4)); diff --git a/src/LibRed/LibRed.Core/Storage/TableCreator.cs b/src/LibRed/LibRed.Core/Storage/TableCreator.cs index 44c92ca13..101183457 100644 --- a/src/LibRed/LibRed.Core/Storage/TableCreator.cs +++ b/src/LibRed/LibRed.Core/Storage/TableCreator.cs @@ -1,9 +1,11 @@ using System.Buffers.Binary; using System.Text; +using EntityFrameworkCore.Jet.Data; using LibRed.Catalog; using LibRed.Formats; using LibRed.IO; using LibRed.Pages; +using MapRetirement = ((int Row, int Page) Map, System.Collections.Generic.IReadOnlyList<(int Row, int Page)> Clear, System.Collections.Generic.IReadOnlyList Pages); namespace LibRed.Storage; @@ -91,6 +93,10 @@ public void Create( // data and a 100-char index name breaks ACE's index enumeration. Only validate caller-supplied names. if (primaryKeyName is not null) JetName.Validate(primaryKeyName, "primary key name"); foreach (RelationshipSpec r in relationships) JetName.Validate(r.Name, "foreign key name"); + var relationshipNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (RelationshipSpec r in relationships) + if (!relationshipNames.Add(r.Name)) throw RelationshipNameTaken(r.Name); + else EnsureRelationshipNameFree(r.Name); foreach (UniqueIndexSpec u in uniqueConstraints) JetName.Validate(u.Name, "unique constraint name"); foreach ((string checkName, _) in checkConstraints) JetName.Validate(checkName, "check constraint name"); @@ -105,6 +111,18 @@ public void Create( throw new InvalidOperationException( $"Table '{name}' has {columns.Count} columns; Jet/ACE tables are limited to {MaxColumnsPerTable}."); + // Before the foreign keys' type match, as ACE checks it: an OLE column referencing a LONG key gets this. + RejectOleIndexColumns( + (primaryKey ?? []).Concat(uniqueConstraints.SelectMany(u => u.Columns)) + .Concat(relationships.SelectMany(r => r.Columns.Select(c => c.Column))), + n => columns.FirstOrDefault(c => string.Equals(c.Name, n, StringComparison.OrdinalIgnoreCase))?.Type); + + relationships = relationships.Select(fk => ResolvePrimaryKeyReference(fk, creatingTable: name)).ToList(); + foreach (RelationshipSpec fk in relationships) + EnsureSameDataTypes(fk, ColumnOf(columns), string.Equals(fk.ReferencedTable, name, StringComparison.OrdinalIgnoreCase) + ? ColumnOf(columns) + : ColumnOf(_catalog.FindTable(fk.ReferencedTable))); + JetFormatBase format = _channel.Format; // Allocate the pages the table needs through the global free-pages map (so Access accounts @@ -263,18 +281,19 @@ public void Create( WriteInt24(tdef, format.TdefFreePagesOffset + 1, usageMapPage); // A wide table's definition can exceed one page; write it split across continuation pages if needed. int defEnd = BinaryPrimitives.ReadInt32LittleEndian(tdef.AsSpan(format.TdefLengthOffset, 4)); - WriteDefinition(tdefPage, tdef[..defEnd], []); + WriteDefinition(tdefPage, tdef[..defEnd], [], rewrite: false); // Per-column extended properties, in column order with DefaultValue before Required (matching ACE): - // a DEFAULT is a memo property; a NOT NULL column carries a boolean Required property (Access omits - // it for a nullable column, and — verified — for an AutoNumber, which is implicitly required). + // a DEFAULT is a memo property; a NOT NULL column carries a boolean Required property, and a nullable + // one none. An AutoNumber follows the same rule — ACE writes Required for COUNTER NOT NULL and not for + // a bare COUNTER (verified by reading its property blob back). var columnProps = new List(); foreach (ColumnSpec col in columns) { var def = columnDefaults.FirstOrDefault(d => string.Equals(d.Column, col.Name, StringComparison.OrdinalIgnoreCase)); if (def.DefaultSql is not null) columnProps.Add(new PropertyBlob.Property(col.Name, PropertyBlob.DefaultValueProperty, def.DefaultSql)); - if (!col.IsNullable && !col.IsAutoNumber) + if (!col.IsNullable) columnProps.Add(PropertyBlob.Bool(col.Name, PropertyBlob.RequiredProperty, true)); columnProps.AddRange(CalculatedProperties(col)); } @@ -338,6 +357,31 @@ private readonly record struct IncomingRelationship( int ParentPage, int Number, int ReferencedOrdinal, uint ChildBlockNumber, int ChildPage, byte UpdateAction, byte DeleteAction); + /// + /// ACE's "same data types" rule for a relationship, measured over every pairing of the column types: each child + /// column must have its parent column's storage type, whatever either one's length — TEXT(5), + /// TEXT(20) and CHAR(10) all pair with one another, as do DECIMALs of any precision and + /// scale and BINARY with VARBINARY. An AutoNumber is a Long on either side. Checked before + /// anything is written. A column that is not found is left to the check that reports it. + /// + private static void EnsureSameDataTypes(RelationshipSpec fk, + Func childColumn, Func parentColumn) + { + foreach ((string column, string referenced) in fk.Columns) + { + if (childColumn(column) is not { } child || parentColumn(referenced) is not { } parent) continue; + if (child != parent) + throw new InvalidOperationException( + "Relationship must be on the same number of fields with the same data types. " + + $"'{column}' ({child}) cannot reference '{fk.ReferencedTable}.{referenced}' ({parent})."); + } + } + + private static Func ColumnOf(IReadOnlyList columns) => + name => columns.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase))?.Type; + + private static Func ColumnOf(TableDef? table) => name => table?.FindColumn(name)?.Type; + /// The data-block ordinal of the index over a self-reference's referenced columns, found /// among the indexes being created for this table (the table is not in the catalog yet). private static int SelfReferencedOrdinal( @@ -384,10 +428,42 @@ private int NextLogicalIndexNumber(int tdefPage) return maxNum + 1; } + /// A relationship's parent table, or ACE's error when it does not exist. + private TableDef ReferencedTableOf(RelationshipSpec fk) => + _catalog.FindTable(fk.ReferencedTable) + ?? throw new InvalidOperationException( + $"Cannot find table or constraint: the referenced table '{fk.ReferencedTable}' does not exist."); + + /// + /// Resolves REFERENCES table with no column list () + /// to the parent's primary key, pairing the child columns with the key's in order whatever either side's + /// columns are named — as ACE does. Any other relationship comes back unchanged. + /// + /// + /// ACE refuses it when the parent has no primary key (a unique index does not stand in for one) and when the + /// columns differ in number (verified). A table referencing itself in its own CREATE TABLE + /// () has no key in the catalog yet; the SQL parser pairs such a reference + /// with a key the statement declares earlier, so one still unpaired here has no key to reference. + /// + private RelationshipSpec ResolvePrimaryKeyReference(RelationshipSpec fk, string? creatingTable) + { + if (!fk.ReferencesPrimaryKey) return fk; + bool selfInCreate = creatingTable is not null + && string.Equals(fk.ReferencedTable, creatingTable, StringComparison.OrdinalIgnoreCase); + IndexDef primaryKey = (selfInCreate ? null : ReferencedTableOf(fk).Indexes.FirstOrDefault(i => i.IsPrimaryKey)) + ?? throw new InvalidOperationException( + $"Cannot create relationship. Referenced table '{fk.ReferencedTable}' does not have a primary key."); + return fk with + { + Columns = RelationshipSpec.PairColumns(fk.ReferencedTable, + fk.Columns.Select(c => c.Column).ToList(), primaryKey.Columns.Select(c => c.Column.Name).ToList()), + ReferencesPrimaryKey = false, + }; + } + private (int Page, int ReferencedOrdinal, int NextIndexNumber) ResolveParent(RelationshipSpec fk, int childPage) { - TableDef parent = _catalog.FindTable(fk.ReferencedTable) - ?? throw new InvalidOperationException($"Referenced table '{fk.ReferencedTable}' was not found."); + TableDef parent = ReferencedTableOf(fk); if (parent.DefinitionPage == childPage) throw new InvalidOperationException($"Self-referencing foreign key '{fk.Name}' should have been handled inline."); @@ -398,15 +474,28 @@ private int NextLogicalIndexNumber(int tdefPage) return (parent.DefinitionPage, refIndex.RealIndexOrdinal, NextLogicalIndexNumber(parent.DefinitionPage)); } + /// ACE refuses a relationship name another relationship already has (verified); a table or query may + /// share it. Checked before anything is written. + private void EnsureRelationshipNameFree(string name) + { + if (_catalog.Relationships.Any(r => string.Equals(r.Name, name, StringComparison.OrdinalIgnoreCase))) + throw RelationshipNameTaken(name); + } + + private static SchemaObjectExistsException RelationshipNameTaken(string name) => + new($"There is already a relationship named '{name}' in the current database.", name); + /// /// Writes the MSysRelationships rows for one relationship — one row per column pair, with /// ccolumn = the pair count, icolumn = the 0-based pair index, and grbit - /// encoding enforce/cascade (verified against Access: an enforced no-cascade FK stores grbit 0). + /// encoding enforce/cascade (verified against Access: an enforced no-cascade FK stores grbit 0) — and the + /// relationship's own MSysObjects object, which ACE records for every relationship. /// private void AddRelationshipRows(string childTable, RelationshipSpec fk) { TableDef msys = _catalog.FindTable("MSysRelationships") ?? throw new InvalidOperationException("MSysRelationships catalog table was not found."); + new ViewCreator(_channel, _catalog).CreateRelationshipObject(fk.Name); int grbit = 0; if (!fk.IsEnforced) grbit |= RelationshipFlags.DontEnforce; @@ -452,7 +541,11 @@ public void AddIndex(string tableName, string indexName, IReadOnlyList<(string C RejectCalculatedIndexColumns(indexName, columns.Select(c => c.Column), n => table.Columns.FirstOrDefault( c => c.IsCalculated && string.Equals(c.Name, n, StringComparison.OrdinalIgnoreCase))?.Name); + RejectOleIndexColumns(columns.Select(c => c.Column), n => table.FindColumn(n)?.Type); var slots = ResolveSlots(table, columns.Select(c => (c.Column, Ascending: !c.Descending))); + // A table has one primary key, and ACE refuses a second (verified). + if (isPrimary && table.Indexes.Any(i => i.IsPrimaryKey)) + throw new InvalidOperationException($"Primary key already exists on table '{table.Name}'."); InsertIndex(table, indexName, slots, unique: isUnique || isPrimary, required: isPrimary || disallowNull, ignoreNulls, (num, ord) => BuildPlainInfoBlock(num, ord, isPrimary)); @@ -476,6 +569,17 @@ private static void RejectCalculatedIndexColumns( + "offer one for indexing, and an index over it makes the table refuse every insert."); } + /// Refuses an index over an OLE column — a key, a unique constraint, a relationship's — before anything + /// is written, as ACE does on every route (verified: CREATE INDEX, PRIMARY KEY and UNIQUE both in CREATE TABLE and + /// added, a foreign key in either place, and ALTER COLUMN of an indexed column to OLE). An OLE value has no index + /// key, and without this the definition is accepted on an empty table and every later insert fails. + private static void RejectOleIndexColumns(IEnumerable columnNames, Func typeOf) + { + foreach (string name in columnNames) + if (typeOf(name) == JetDataType.Ole) + throw new InvalidOperationException($"Invalid field definition '{name}' in definition of index or relationship."); + } + /// Resolves index column names to (columnId, ascending) slots against a table. private static IReadOnlyList<(int Id, bool Ascending)> ResolveSlots( TableDef table, IEnumerable<(string Column, bool Ascending)> columns) @@ -509,13 +613,13 @@ private int InsertIndex(TableDef table, string indexName, IReadOnlyList<(int Id, (LibRed.IO.PageBuffer buf, IReadOnlyList existingContinuations) = ReadDefinition(table.DefinitionPage); int existingRowCount = buf.ReadInt32(format.TdefRowCountOffset); - // A unique index over rows that already exist has to be rejected if those rows aren't unique — ACE - // refuses the DDL too. Done *here*, before a single byte of the TDEF moves, rather than during the - // back-fill: this path is not transactional, so a failure discovered mid-back-fill would leave the - // index committed to the TDEF and half-populated. Scanning first means a rejected CREATE INDEX - // leaves the file exactly as it was. - if (unique && existingRowCount != 0) - EnsureNoDuplicateKeys(table, indexName, slots); + // A unique index over rows that already exist has to be rejected if those rows aren't unique, and a + // required one if any row leaves a key column NULL — ACE refuses the DDL for both. Done *here*, before a + // single byte of the TDEF moves, rather than during the back-fill: this path is not transactional, so a + // failure discovered mid-back-fill would leave the index committed to the TDEF and half-populated. + // Scanning first means a rejected CREATE INDEX leaves the file exactly as it was. + if ((unique || required) && existingRowCount != 0) + EnsureExistingRowsFitIndex(table, indexName, slots, unique, required); int dataCount = buf.ReadInt32(format.TdefIndexCountOffset); int logicalCount = buf.ReadInt32(format.TdefLogicalIndexCountOffset); @@ -627,7 +731,7 @@ private int InsertIndex(TableDef table, string indexName, IReadOnlyList<(int Id, System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLogicalIndexCountOffset, 4), logicalCount + 1); System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLengthOffset, 4), newDefEnd); - WriteDefinition(table.DefinitionPage, def, existingContinuations); + WriteDefinition(table.DefinitionPage, def, existingContinuations, rewrite: true); _catalog.Invalidate(); // Back-fill the new (empty) index B-tree with an entry per existing row, so the index is complete. @@ -637,12 +741,14 @@ private int InsertIndex(TableDef table, string indexName, IReadOnlyList<(int Id, } /// - /// Throws if the table's existing rows already hold a duplicate key for a would-be unique index. Purely a - /// read: it encodes each row's key and looks for a repeat, touching nothing on disk, so it is safe to call - /// before the index exists. Rows with a null in any key column are exempt — Jet's uniqueness is over the - /// non-null keys only, so several rows may be null (verified vs ACE; see the constraints page). That skip - /// is unconditional: a WITH IGNORE NULL index leaves null-keyed rows out of the B-tree altogether, so - /// either way they cannot collide. + /// Throws if the table's existing rows cannot go into a would-be index: a duplicate key for a unique one, or + /// a NULL in any key column for a required one — a primary key or WITH DISALLOW NULL, which ACE refuses with + /// "Index or primary key cannot contain a Null value" (verified; an ADD COLUMN … PRIMARY KEY on a table that + /// already holds rows is the usual way to get there). Purely a read, touching nothing on disk, so it is safe + /// to call before the index exists. For uniqueness, rows with a null in any key column are exempt — Jet's + /// uniqueness is over the non-null keys only, so several rows may be null (verified vs ACE; see the + /// constraints page); a WITH IGNORE NULL index leaves them out of the B-tree altogether, so either way they + /// cannot collide. /// /// /// Comparison is on the encoded key, not the raw values, which is deliberate: the encoding is what the @@ -650,8 +756,8 @@ private int InsertIndex(TableDef table, string indexName, IReadOnlyList<(int Id, /// uniqueness domain, and it keeps this agreeing with the insert- and update-time checks, which compare /// the same way via . /// - private void EnsureNoDuplicateKeys( - TableDef table, string indexName, IReadOnlyList<(int Id, bool Ascending)> slots) + private void EnsureExistingRowsFitIndex( + TableDef table, string indexName, IReadOnlyList<(int Id, bool Ascending)> slots, bool unique, bool required) { var keyColumns = slots .Select(s => (Column: table.Columns.First(c => c.ColumnId == s.Id), s.Ascending)) @@ -660,8 +766,14 @@ private void EnsureNoDuplicateKeys( var seen = new HashSet(); foreach (object?[] values in new Table(_channel, table).Rows()) { - if (keyColumns.Any(k => values[k.Column.Index] is null)) continue; - if (!seen.Add(Convert.ToHexString(IndexKeyEncoder.Encode(keyColumns, values)))) + if (keyColumns.Any(k => values[k.Column.Index] is null)) + { + if (required) + throw new InvalidOperationException( + $"Index or primary key cannot contain a Null value: a row of '{table.Name}' has no value for index '{indexName}'."); + continue; + } + if (unique && !seen.Add(Convert.ToHexString(IndexKeyEncoder.Encode(keyColumns, values)))) throw new InvalidOperationException( $"Cannot create unique index '{indexName}' on '{table.Name}': duplicate key values exist."); } @@ -676,15 +788,27 @@ private void EnsureNoDuplicateKeys( /// replaced by the next entry. Sorting first lets each leaf be filled and written once. /// Uniqueness moves with it: on sorted keys a duplicate is an adjacent pair, so the per-row /// descent is gone. The comparison is still on the encoded key, which - /// is Access's uniqueness domain (see ). + /// is Access's uniqueness domain (see ). + /// A built index's statistics are set as ACE sets them (verified, for CREATE INDEX, a foreign key's + /// backing index and an ALTER COLUMN's rebuild): the total entry count to the entries it now holds and the + /// unique entry count to its distinct keys — both from the rows present, not from any earlier history. /// private void BackfillIndex(string tableName, string indexName, bool ignoreNulls, bool validateUnique) { TableDef table = _catalog.FindTable(tableName) ?? throw new InvalidOperationException($"Table '{tableName}' was not found after adding the index."); IndexDef index = table.Indexes.First(ix => string.Equals(ix.Name, indexName, StringComparison.OrdinalIgnoreCase)); - var keyColumnIds = index.Columns.Select(c => c.Column.Index).ToArray(); + List<(byte[] Key, int Pointer, bool NullKey)> entries = IndexEntries(table, index, ignoreNulls); + new IndexWriter(_channel, table).BulkBuild(index, entries, validateUnique && index.IsUnique); + SetBuiltStatistics(table, index, entries); + } + + /// The entries an index over the table's current rows holds: every live row's encoded key and row + /// pointer, less the rows an IGNORE NULL index leaves out. + private List<(byte[] Key, int Pointer, bool NullKey)> IndexEntries(TableDef table, IndexDef index, bool ignoreNulls) + { + var keyColumnIds = index.Columns.Select(c => c.Column.Index).ToArray(); var entries = new List<(byte[] Key, int Pointer, bool NullKey)>(); foreach ((RowId id, object?[] values) in new Table(_channel, table).Rows().WithIds()) { @@ -692,8 +816,36 @@ private void BackfillIndex(string tableName, string indexName, bool ignoreNulls, if (ignoreNulls && hasNullKey) continue; entries.Add((IndexKeyEncoder.Encode(index.Columns, values), (id.Page << 8) | id.Row, hasNullKey)); } + return entries; + } - new IndexWriter(_channel, table).BulkBuild(index, entries, validateUnique && index.IsUnique); + /// Sets a just-built index's statistics block as ACE sets it: total = the entries it holds, unique = + /// its distinct keys among them. + private void SetBuiltStatistics(TableDef table, IndexDef index, List<(byte[] Key, int Pointer, bool NullKey)> entries) => + WriteIndexStatistics(table, index, entries.Count, entries.Select(e => Convert.ToHexString(e.Key)).Distinct().Count()); + + /// Every logical index's statistics — its real index's total and unique entry counts — by index name. + /// The blocks sit on the definition's first page. + private Dictionary IndexStatistics(TableDef table) + { + byte[] tdef = _channel.ReadPage(table.DefinitionPage).Span.ToArray(); + var statistics = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (IndexDef index in table.Indexes) + { + int at = _channel.Format.TdefRealIndexBlockOffset + index.RealIndexOrdinal * _channel.Format.RealIndexEntrySize; + statistics[index.Name] = (BinaryPrimitives.ReadInt32LittleEndian(tdef.AsSpan(at, 4)), + BinaryPrimitives.ReadInt32LittleEndian(tdef.AsSpan(at + 4, 4))); + } + return statistics; + } + + private void WriteIndexStatistics(TableDef table, IndexDef index, int total, int unique) + { + byte[] tdef = _channel.ReadPage(table.DefinitionPage).Span.ToArray(); + int at = _channel.Format.TdefRealIndexBlockOffset + index.RealIndexOrdinal * _channel.Format.RealIndexEntrySize; + BinaryPrimitives.WriteInt32LittleEndian(tdef.AsSpan(at, 4), total); + BinaryPrimitives.WriteInt32LittleEndian(tdef.AsSpan(at + 4, 4), unique); + _channel.WritePage(table.DefinitionPage, tdef); } /// Appends one empty inline usage-map record (row ) to an existing @@ -750,12 +902,17 @@ private void AppendEmptyUsageMapRow(JetFormatBase format, int pageNumber, int ne public void AddForeignKey(string childTable, RelationshipSpec fk) { JetName.Validate(fk.Name, "foreign key name"); + EnsureRelationshipNameFree(fk.Name); TableDef child = _catalog.FindTable(childTable) ?? throw new InvalidOperationException($"Table '{childTable}' was not found."); if (fk.NoIndex) throw new NotSupportedException("ALTER TABLE ADD FOREIGN KEY … NO INDEX is not supported yet."); if (fk.UpdateSetNull) throw UpdateSetNullNotImplemented(); + RejectOleIndexColumns(fk.Columns.Select(c => c.Column), n => child.FindColumn(n)?.Type); + fk = ResolvePrimaryKeyReference(fk, creatingTable: null); + EnsureSameDataTypes(fk, ColumnOf(child), ColumnOf(_catalog.FindTable(fk.ReferencedTable))); + byte upd = fk.CascadeUpdate ? CascadeAction : NoCascadeAction; byte del = fk.CascadeDelete ? CascadeAction : fk.DeleteSetNull ? SetNullAction : NoCascadeAction; var slots = ResolveSlots(child, fk.Columns.Select(c => (c.Column, Ascending: true))); @@ -829,10 +986,17 @@ public bool DropConstraint(string childTable, string name) WriteTdef(parent.DefinitionPage, parentParts); } - new PageAllocator(_channel).Free(fkIndex.RootPage); + new PageAllocator(_channel).Release(fkIndex.RootPage); } SoftDeleteRelationshipRows(name); + // Its MSysObjects object and permission rows go with it, as ACE removes them (verified). A relationship + // written before LibRed recorded the object has none. + if (FindObjectId(name, CatalogFormat.ObjectTypeRelationship) is { } objectId) + { + DeleteCatalogRows("MSysObjects", "Id", objectId); + DeleteCatalogRows("MSysACEs", "ObjectId", objectId); + } _catalog.Invalidate(); return true; } @@ -840,8 +1004,9 @@ public bool DropConstraint(string childTable, string name) /// /// Drops a table — DROP TABLE table. Removes the object's MSysObjects row and its /// MSysACEs permission rows (soft-delete, as ACE does), and frees the table's pages back to the - /// global free map so a later create reuses them (verified vs ACE): its index B-tree roots, its data - /// pages (owned-pages usage map), and the TDEF page. Returns false if the table doesn't exist. + /// global free map when the database closes (verified vs ACE): every page of its indexes, its data and + /// long-value pages, the TDEF page and its continuation pages, a reference-form map's bitmap pages, and any + /// usage-map holder left with no live record. Returns false if the table doesn't exist. /// /// A table that is the child (referencing) side of relationships can be dropped directly: ACE /// lets you drop the referencing table while the parent stays, so each such relationship is removed first @@ -849,9 +1014,6 @@ public bool DropConstraint(string childTable, string name) /// child cannot be dropped — drop the referencing table (or the relationship) first. EF drops FKs before /// tables, but database-first scaffolding cleanup drops child tables directly, which must work. /// - /// Multi-page TDEFs, multi-level index trees (non-root pages), memo/OLE LVAL pages and dedicated - /// usage-map pages are not yet freed (they leak until Compact); the catalog removal is complete regardless, - /// so the table disappears and Access opens the file. public bool DropTable(string tableName) { TableDef? table = _catalog.FindTable(tableName); @@ -876,11 +1038,68 @@ public bool DropTable(string tableName) int tdefPage = table.DefinitionPage; var allocator = new PageAllocator(_channel); + var maps = new UsageMap(_channel, table); + + // Collect before freeing anything: the pointers are read out of the TDEF, which this method frees. + var owned = new HashSet(); + // Through the chain reader: a wide table's definition spans continuation pages, and parsing only the + // first one throws on the declared length. (The map POINTERS below are at fixed offsets inside the + // first page, so those are read from it directly, as UsageMap does.) + var definition = new TableDefinitionPage(); + definition.Read(_channel, tdefPage); + + // The map RECORDS live as rows on owner-zero data pages, and each is retired in turn — its bits cleared + // first where ACE clears them, then its row reclaimed, which slides every record below it up the page. + // The order is ACE's and it shows on disk: a slide leaves a copy of the records it moved in the space + // they vacated, so retiring the same records in another order leaves different bytes behind. Measured + // by whole-file diff against ACE drops: each long-value column's owned then free map, bits cleared; + // then each index's owned map in index order, bits cleared; then the table's own owned map, bits + // cleared, and its free map, whose bits stay. + var retire = new List(); + + foreach (ColumnDef column in table.Columns) + QueueLongValueMaps(definition, column, maps, owned, retire); + + // Each real index keeps its B-tree pages in its own owned map, whose (row, page) pointer sits in its data + // block. Freeing only the root strands every other page of a multi-level index, and leaving the map's + // record live keeps its holder from ever being freed. + foreach (byte[] block in ParseTdef(tdefPage).DataBlocks) + { + int at = IndexBlockFormat.UsageMapRowOffset; + (int Row, int Page) map = (block[at], block[at + 1] | block[at + 2] << 8 | block[at + 3] << 16); + if (map.Page == 0) continue; + List pages = maps.PagesInMap(map.Row, map.Page).ToList(); + owned.UnionWith(pages); + retire.Add((map, [map], pages)); + } foreach (IndexDef index in table.Indexes.Where(i => i.RootPage > 0).GroupBy(i => i.RootPage).Select(g => g.First())) - allocator.Free(index.RootPage); - foreach (int dataPage in new UsageMap(_channel, table).DataPages()) - allocator.Free(dataPage); - allocator.Free(tdefPage); + owned.Add(index.RootPage); + List dataPages = maps.DataPages().ToList(); + owned.UnionWith(dataPages); + owned.Add(tdefPage); + // A wide table's definition continues on further pages. ACE frees them too, and leaves every byte of + // them alone — only the first page is marked released. + owned.UnionWith(TdefChainReader.Read(_channel, tdefPage).ContinuationPages); + + PageBuffer tdef = _channel.ReadPage(tdefPage); + (int Row, int Page) dataOwned = (tdef.ReadByte(_channel.Format.TdefOwnedPagesOffset), + tdef.ReadInt24(_channel.Format.TdefOwnedPagesOffset + 1)); + retire.Add((dataOwned, [dataOwned], dataPages)); + retire.Add(((tdef.ReadByte(_channel.Format.TdefFreePagesOffset), + tdef.ReadInt24(_channel.Format.TdefFreePagesOffset + 1)), [], [])); + + RetireMapRecords(retire, maps, owned); + + // Access marks the released definition page itself: its type byte becomes 0x08 and nothing else on + // the page changes, so the old definition is still sitting there when Compact comes to reclaim it. + // Measured across an ACE DROP TABLE: exactly one byte of the 4,096 differs. Only the TDEF is marked — + // the data, long-value and map-holder pages ACE frees keep their 0x01. + byte[] released = _channel.ReadPage(tdefPage).Span.ToArray(); + released[0] = (byte)PageType.ReleasedTableDefinition; + _channel.WritePage(tdefPage, released); + + foreach (int page in owned) + allocator.Release(page); // reusable only after this handle closes, as ACE holds them DeleteCatalogRows("MSysObjects", "Id", tdefPage); DeleteCatalogRows("MSysACEs", "ObjectId", tdefPage); @@ -888,6 +1107,84 @@ public bool DropTable(string tableName) return true; } + /// + /// Queues a long-value column's two usage-map records for , owned then free, and + /// adds the pages its owned map records to . A Memo/OLE (or calculated long) column owns + /// its LVAL pages through a PER-COLUMN usage map, whose pointer sits in the TDEF keyed by column id. Those pages + /// are not in the table's data-page map, so freeing only the data pages leaves every long value stranded — and + /// for a memo-heavy table that is nearly the whole table. Measured against ACE: dropping a 60-row memo table + /// returned 123 pages through ACE and 2 through LibRed, the missing 121 being LVAL pages. Nothing is queued for + /// a column with no long-value maps. + /// + private static void QueueLongValueMaps( + TableDefinitionPage definition, ColumnDef column, UsageMap maps, HashSet owned, List retire) + { + definition.LongValueOwnedMaps.TryGetValue(column.ColumnId, out (int Row, int Page) map); + definition.LongValueFreeMaps.TryGetValue(column.ColumnId, out (int Row, int Page) columnFree); + if (map.Page != 0) + { + // Clear each page's bit on the way out, exactly as releasing a single value does: the record's + // bitmap bytes are zeroed before its row is retired. Except a page still in the free map, whose bit + // stays in both records — measured by whole-file diff of ACE drops: of an OLE column owning a chain, + // two full single-value pages and its current append page, every bit went but the append page's. + List pages = maps.PagesInMap(map.Row, map.Page).ToList(); + owned.UnionWith(pages); + HashSet stillFree = columnFree.Page != 0 ? maps.PagesInMap(columnFree.Row, columnFree.Page).ToHashSet() : []; + retire.Add((map, columnFree.Page != 0 ? [map, columnFree] : [map], pages.Where(p => !stillFree.Contains(p)).ToList())); + } + if (columnFree.Page != 0) retire.Add((columnFree, [], [])); + } + + /// + /// Takes usage-map records off their pages the way ACE does, in the order given — tombstone the slot, slide the + /// rows below it up, return the bytes to the page's free count — rather than leaving dead maps behind. On a + /// shared holder that is the whole fix: the page survives and must not keep records for a map that no longer + /// exists. Each record's Clear maps first have its Pages cleared. Adds to + /// the pages to release: a reference-form record's bitmap pages, and every holder left with no live row. + /// + private void RetireMapRecords(IEnumerable retire, UsageMap maps, HashSet owned) + { + var usageMaps = new UsageMapWriter(_channel); + var holders = new List(); + var retired = new HashSet<(int Row, int Page)>(); + foreach (((int Row, int Page) map, IReadOnlyList<(int Row, int Page)> clear, IReadOnlyList pages) in retire) + { + if (map.Page <= 1 || map.Page >= _channel.PageCount || !retired.Add(map)) continue; + foreach ((int Row, int Page) cleared in clear) + foreach (int page in pages) + usageMaps.SetBit(cleared.Row, cleared.Page, page, set: false); + + // A reference-form record keeps its bitmap on dedicated pages. ACE zeroes each one's bitmap — even + // for the table's own owned map, whose bits an inline record keeps — leaves its header, and frees it. + foreach (int bitmapPage in maps.BitmapPagesOf(map.Row, map.Page)) + { + byte[] bitmap = _channel.ReadPage(bitmapPage).Span.ToArray(); + bitmap.AsSpan(4).Clear(); + _channel.WritePage(bitmapPage, bitmap); + owned.Add(bitmapPage); + } + + byte[] holderBytes = _channel.ReadPage(map.Page).Span.ToArray(); + RowInserter.ReclaimRow(_channel.Format, holderBytes, map.Row); + _channel.WritePage(map.Page, holderBytes); + if (!holders.Contains(map.Page)) holders.Add(map.Page); + } + + // ACE frees a holder once the dropped records were the only thing on it — measured: for a one-memo-column + // table ACE returned the long-value map's holder. A holder can carry records for several columns or tables + // as separate rows, so releasing one that still serves another map would hand away a live page: corruption + // rather than a leak. Hence the holder goes only when no live row is left on it. + foreach (int holderPage in holders) + { + var holder = new DataPage(); + holder.Read(_channel.ReadPage(holderPage), _channel.Format); + bool live = false; + for (int row = 0; row < holder.RowCount && !live; row++) + live = !holder.Rows[row].IsDeleted && holder.Rows[row].Length > 0; + if (!live) owned.Add(holderPage); + } + } + /// /// Drops a view or stored procedure — DROP VIEW name / DROP PROCEDURE name. Both are a /// type-5 MSysObjects object; ACE's two statements are interchangeable (verified: DROP VIEW works @@ -897,6 +1194,19 @@ public bool DropTable(string tableName) /// shared MSysQueries pages). Returns false if no such query object exists. /// public bool DropQueryObject(string name) + { + if (FindObjectId(name, StoredQueryFormat.ObjectTypeQuery) is not { } objId) return false; + + DeleteCatalogRows("MSysObjects", "Id", objId); + DeleteCatalogRows("MSysQueries", "ObjectId", objId); + DeleteCatalogRows("MSysACEs", "ObjectId", objId); + _catalog.Invalidate(); + return true; + } + + /// The MSysObjects id of the object of named , + /// or null when there is none. + private int? FindObjectId(string name, short type) { TableDef mo = _catalog.FindTable("MSysObjects") ?? throw new InvalidOperationException("MSysObjects catalog table was not found."); @@ -904,18 +1214,11 @@ public bool DropQueryObject(string name) int nameIdx = (mo.FindColumn("Name") ?? throw new InvalidOperationException("MSysObjects is missing 'Name'.")).Index; int typeIdx = (mo.FindColumn("Type") ?? throw new InvalidOperationException("MSysObjects is missing 'Type'.")).Index; - int? objId = null; foreach (object?[] values in new Table(_channel, mo).Rows()) if (string.Equals(values[nameIdx] as string, name, StringComparison.OrdinalIgnoreCase) - && Convert.ToInt16(values[typeIdx] ?? (short)0) == StoredQueryFormat.ObjectTypeQuery) - { objId = Convert.ToInt32(values[idIdx]); break; } - if (objId is null) return false; - - DeleteCatalogRows("MSysObjects", "Id", objId.Value); - DeleteCatalogRows("MSysQueries", "ObjectId", objId.Value); - DeleteCatalogRows("MSysACEs", "ObjectId", objId.Value); - _catalog.Invalidate(); - return true; + && Convert.ToInt16(values[typeIdx] ?? (short)0) == type) + return Convert.ToInt32(values[idIdx]); + return null; } @@ -1256,7 +1559,7 @@ public bool DropIndex(string tableName, string indexName) RemoveTdefBlocks(parts, removeDataOrdinal: index.RealIndexOrdinal, removeLogical: b => NameOf(b.Name).Equals(indexName, StringComparison.OrdinalIgnoreCase)); WriteTdef(table.DefinitionPage, parts); - new PageAllocator(_channel).Free(index.RootPage); + new PageAllocator(_channel).Release(index.RootPage); _catalog.Invalidate(); return true; } @@ -1442,9 +1745,44 @@ public bool AddColumn(string tableName, ColumnSpec spec, string? defaultValue = if (props.Count > 0) SetColumnProperties(table.DefinitionPage, spec.Name, props); _catalog.Invalidate(); + if (spec.IsAutoNumber) NumberExistingRows(tableName, spec); return true; } + /// + /// Gives the rows already in a table values in an AutoNumber column just added to it, as ACE does rather than + /// leaving them NULL, and sets where the counter carries on (all verified). + /// + /// + /// The existing rows are numbered 1, 2, 3 … in table order whatever the column's seed and increment. A counter + /// with the default seed 1 and increment 1 — however it was spelled — then continues after them: two rows take + /// 1 and 2, the next insert 3. Any other counter restarts at its own seed, even where that repeats a value the + /// rows were given: COUNTER(2, 1) over two rows goes on 2, 3, 4, and COUNTER(1, 5) 1, 6, 11. A + /// primary key over the new column has a value in every row either way. + /// + private void NumberExistingRows(string tableName, ColumnSpec spec) + { + TableDef table = _catalog.FindTable(tableName) + ?? throw new InvalidOperationException($"Table '{tableName}' was not found after adding column '{spec.Name}'."); + ColumnDef column = table.FindColumn(spec.Name)!; + int increment = spec.Increment == 0 ? 1 : spec.Increment; + + var rows = new Table(_channel, table).Rows().WithIds().ToList(); + if (rows.Count > 0) + { + var writer = new RowInserter(_channel, table); + var changed = new HashSet { column.Index }; + int number = 0; + foreach ((RowId id, object?[] values) in rows) + { + values[column.Index] = ++number; + writer.Update(id, values, changed); + } + } + + ReseedCounter(table, column, spec.Seed == 1 && increment == 1 ? rows.Count + 1 : spec.Seed, increment); + } + /// Inserts a long-value (memo/OLE) column's 10-byte §3.3.2 usage-map entry /// ({col_num:2}{used row+page:4}{free row+page:4}) just before the list's 0xFFFF terminator. /// The new column has the highest id, so appending keeps the list in ascending column order. @@ -1465,6 +1803,22 @@ private static void AddLongValueMapEntry(TdefParts parts, int columnId, int used parts.Lval = result; } + /// Removes a long-value column's 10-byte §3.3.2 usage-map entry from the list, keeping the other + /// entries and the 0xFFFF terminator. A no-op for a column without one. + private static void RemoveLongValueMapEntry(TdefParts parts, int columnId) + { + byte[] lval = parts.Lval; + for (int at = 0; at + 2 < lval.Length; at += 10) + { + if (BinaryPrimitives.ReadUInt16LittleEndian(lval.AsSpan(at, 2)) != columnId) continue; + var result = new byte[lval.Length - 10]; + Array.Copy(lval, 0, result, 0, at); + Array.Copy(lval, at + 10, result, at, lval.Length - at - 10); + parts.Lval = result; + return; + } + } + /// Sets (replaces) a column's DefaultValue in the table's MSysObjects.LvProp blob — /// ALTER TABLE … ALTER COLUMN … DEFAULT. Reads all properties, drops any existing DefaultValue for the /// column, adds the new one, and rewrites the blob (preserving every other property). @@ -1927,6 +2281,12 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo // 1. Materialise all rows (values indexed by column position) before dropping the table. var rows = new Table(_channel, def).Rows().Select(r => (object?[])r.Clone()).ToList(); + // Every index's statistics, here and on each table referencing this one: the rebuild re-inserts the rows and + // re-creates the indexes, which would recount them all, where ACE's ALTER leaves every index alone but the + // ones over the column it changes (verified, for Memo/OLE retypes as for the rest). + var statistics = new[] { tableName }.Concat(incoming.Select(r => r.Table)).Distinct(StringComparer.OrdinalIgnoreCase) + .ToDictionary(t => t, t => IndexStatistics(_catalog.FindTable(t)!), StringComparer.OrdinalIgnoreCase); + // 2. Reconstruct the schema — column order preserved, the target re-typed. Every OTHER column keeps its // original descriptor bytes (RawDescriptor passthrough), so fields LibRed doesn't model survive the // rewrite (the faithful round-trip rule); the target builds fresh (RawDescriptor null). Column ids stay @@ -1937,7 +2297,8 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo var specs = def.Columns.Select(c => c.Index == targetIndex ? newColumnSpec with { IsNullable = target.IsNullable, RawDescriptor = null } : new ColumnSpec(c.Name, c.Type, c.Length, c.IsFixedLength, c.IsAutoNumber, c.Precision, c.Scale, - c.IsNullable, c.Seed, c.Increment, RawDescriptor: c.RawDescriptor)).ToList(); + c.IsNullable, c.Seed, c.Increment, RawDescriptor: c.RawDescriptor, + CalculatedExpression: c.CalculatedExpression, CalculatedResultType: c.CalculatedResultType)).ToList(); IndexDef? pk = def.Indexes.FirstOrDefault(i => i.IsPrimaryKey); IReadOnlyList? primaryKey = pk?.Columns.Select(c => c.Column.Name).ToList(); @@ -1954,7 +2315,7 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo // 3. Pre-check: convert every target value in memory BEFORE touching disk. An unconvertible value // (e.g. non-numeric text → INT) throws here, with nothing written — the caller sees a clean failure. foreach (object?[] row in rows) - row[targetIndex] = ConvertValue(row[targetIndex], newColumnSpec.Type); + row[targetIndex] = ConvertValue(row[targetIndex], newColumnSpec.Type, newColumnSpec.Name); // 4. Apply the rebuild atomically: wrap it in a page-level transaction so any failure that slips past the // pre-check (a unique-index collision after narrowing, NOT NULL, an I/O or allocation error) rolls the @@ -1982,7 +2343,17 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo _catalog.Invalidate(); var dest = new Table(_channel, _catalog.FindTable(tableName)!); - foreach (object?[] row in rows) dest.Insert(row); + int[] calculatedColumns = dest.Definition.Columns + .Where(c => c.IsCalculated) + .Select(c => c.Index) + .ToArray(); + foreach (object?[] row in rows) + { + // A calculated value is a cache, not caller-supplied data. Recompute it from its expression + // while rebuilding, as an ordinary insert does; reinserting the old cache is rejected. + foreach (int index in calculatedColumns) row[index] = null; + dest.Insert(row); + } // Restore each index AS IT WAS. IgnoreNulls and Required are read off the 0x2E flags word and are // right here on the IndexDef; hard-coding them false made an ALTER COLUMN on an unrelated column @@ -2013,6 +2384,22 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo _catalog.Invalidate(); } + // Put the statistics back as ACE leaves them: every index as it was, except one over the changed column, + // which ACE rebuilds — counted from the rows it now holds, the primary key too (which the re-insert + // counted as inserts). + foreach ((string name, Dictionary before) in statistics) + { + TableDef table = _catalog.FindTable(name)!; + foreach (IndexDef index in table.Indexes) + { + if (string.Equals(name, tableName, oic) + && index.Columns.Any(c => string.Equals(c.Column.Name, columnName, oic))) + SetBuiltStatistics(table, index, IndexEntries(table, index, index.IgnoreNulls)); + else if (before.TryGetValue(index.Name, out (int Total, int Unique) kept)) + WriteIndexStatistics(table, index, kept.Total, kept.Unique); + } + } + if (ownTransaction) _channel.CommitTransaction(); } catch when (ownTransaction) @@ -2023,30 +2410,6 @@ private void RewriteColumn(string tableName, string columnName, ColumnSpec newCo } } - /// The TDEF-page step of the in-place column type change: bump the 0x29 high-water and rewrite - /// ONLY the target descriptor (type, burned id, fixed-offset appended to the end of the fixed region with the - /// old slot left dead, length), leaving the TDEF page number and every other descriptor byte-identical to ACE. - /// This alone is not a self-consistent change — wraps it with the row - /// re-lay and index rebuild; this entry point exists so a byte-diff test can isolate the TDEF page. - public void AlterColumnTypeInPlaceTdef(string tableName, string columnName, ColumnSpec newSpec, int? fixedEndOverride = null) - { - TableDef def = _catalog.FindTable(tableName) - ?? throw new InvalidOperationException($"Table '{tableName}' does not exist."); - ColumnDef target = def.FindColumn(columnName) - ?? throw new InvalidOperationException($"Column '{columnName}' does not exist in '{tableName}'."); - EnsureColumnIsNotInRelationship(def, target); - JetFormatBase format = _channel.Format; - - TdefParts parts = ParseTdef(def.DefinitionPage); - // The fixed-region end must include dead slots, so callers with rows pass the row-derived length. - int fixedEnd = fixedEndOverride ?? def.Columns.Where(c => c.IsFixedLength && c.Type != JetDataType.Boolean) - .Select(c => c.FixedOffset + c.Length).DefaultIfEmpty(0).Max(); - - EditTargetDescriptor(parts, target, newSpec, fixedEnd, _collation, format); - WriteTdef(def.DefinitionPage, parts); - _catalog.Invalidate(); - } - /// Applies ACE's in-place column retype to the target descriptor within /// (no page write — the caller writes the TDEF once): the target becomes a NEW column with a fresh id from the /// 0x29 high-water and its fixed data appended to the END of the current fixed region (its old slot left @@ -2089,9 +2452,9 @@ private static int EditTargetDescriptor(TdefParts parts, ColumnDef target, Colum return maxCols; } - /// Full in-place column type change, byte-for-byte like ACE (currently: an all-fixed, non-boolean - /// table whose target stays fixed and is not indexed; falls back to otherwise). - /// Edits the TDEF in place () and re-lays every row — the target's + /// Full in-place column type change, byte-for-byte like ACE for fixed and variable columns and + /// targets, fixed↔variable, and indexed targets; a Memo/OLE source or target falls back to + /// . Edits the TDEF in place () and re-lays every row — the target's /// OLD fixed slot is kept as dead space, its converted value appended at the new offset, count + null bitmap /// updated. Converts values in memory first (throws on bad data before any write); runs in a transaction. public void AlterColumnTypeInPlace(string tableName, string columnName, ColumnSpec newSpec) @@ -2101,6 +2464,8 @@ public void AlterColumnTypeInPlace(string tableName, string columnName, ColumnSp ColumnDef oldTarget = oldDef.FindColumn(columnName) ?? throw new InvalidOperationException($"Column '{columnName}' does not exist in '{tableName}'."); EnsureColumnIsNotInRelationship(oldDef, oldTarget); + if (newSpec.Type == JetDataType.Ole && oldDef.Indexes.Any(i => i.Columns.Any(c => c.Column.ColumnId == oldTarget.ColumnId))) + RejectOleIndexColumns([oldTarget.Name], _ => JetDataType.Ole); // Also reached directly, not only through AlterColumn, so it carries the width limits itself. // The record-fits check needs the true fixed-region end, so it runs once that is measured, below. @@ -2127,7 +2492,8 @@ public void AlterColumnTypeInPlace(string tableName, string columnName, ColumnSp var rows = new Table(_channel, oldDef).Rows().WithIds() .Select(r => (r.Id, Raw: reader.ReadRow(r.Id), Values: (object?[])r.Values.Clone())) .ToList(); - foreach (var r in rows) r.Values[oldTarget.Index] = ConvertValue(r.Values[oldTarget.Index], newSpec.Type); + foreach (var r in rows) + r.Values[oldTarget.Index] = ConvertValue(r.Values[oldTarget.Index], newSpec.Type, newSpec.Name); // The fixed-region length is authoritative from the existing rows (their var-data-start), NOT the live // column descriptors — those diverge once a high-offset column has been retyped to variable and left a @@ -2202,7 +2568,7 @@ public void AlterColumnTypeInPlace(string tableName, string columnName, ColumnSp foreach (var p in pending) { BackfillIndex(tableName, p.Name, p.IgnoreNulls, validateUnique: true); - _allocator.Free(p.OldRoot); + _allocator.Release(p.OldRoot); } if (ownTx) _channel.CommitTransaction(); @@ -2291,7 +2657,7 @@ private int PrepareIndexRebuild(TdefParts parts, TableDef table, IndexDef index, int newRow = RecycleOwnedMapRow(format, usageMapPage, oldUsageRow, newRoot); // Re-point the index-data block: the target's burned id in its column slot, the new root, the new - // usage-map row; bump the stats block (+0x00, observed 0→1 on ACE's rebuild). + // usage-map row. Its statistics are set by the backfill that follows, from the rows it then holds. for (int slot = 0; slot < IndexBlockFormat.MaxColumns; slot++) { int at = IndexBlockFormat.ColumnsOffset + slot * IndexBlockFormat.ColumnSlotSize; @@ -2300,48 +2666,120 @@ private int PrepareIndexRebuild(TdefParts parts, TableDef table, IndexDef index, } block[IndexBlockFormat.UsageMapRowOffset] = (byte)newRow; BinaryPrimitives.WriteInt32LittleEndian(block.Slice(IndexBlockFormat.RootPageOffset, 4), newRoot); - Span stats = parts.Stats[index.RealIndexOrdinal]; - BinaryPrimitives.WriteInt32LittleEndian(stats, BinaryPrimitives.ReadInt32LittleEndian(stats) + 1); return newRoot; } - /// Recycles an index's owned-pages usage-map row exactly the way ACE does on a rebuild: append a - /// fresh row and set the new root's bit (ACE's first write, at the appended slot), then MOVE that map into - /// the old row's freed slot and soft-delete the old row (a 0-length deleted+overflow tombstone) — leaving - /// the appended slot's bytes stale in free space, byte-for-byte as ACE does. Returns the new row number. + /// + /// Recycles an index's owned-pages usage-map row the way ACE does on a rebuild, in the two writes whose + /// combined result is observable on disk: (1) append a fresh row at the bottom of the holder page + /// and set the new root's bit — those bytes are then abandoned and stay as a stale copy; (2) lay + /// the page out again with the old row's record reclaimed: its slot becomes a 0-length + /// deleted+overflow tombstone at the preceding record's offset, every later row keeps its number while its + /// record slides up, and the fresh map takes the position freed at the end of the live region under the + /// appended row number. Returns that number — the only pointer the caller re-points, because no other + /// row's number changes and each one's data travels with it. + /// + /// + /// Both halves are load-bearing and each was missed once. The stale copy decides a whole-file byte diff + /// against ACE on a single byte (the new root's bit, at offset 49 of the abandoned record) and is + /// invisible to free-space accounting, since it lies below the lowest live record inside the region free + /// space already covers. The compaction is invisible to slot offsets alone and shows up only when records + /// are identified by content — ACE's own page, before → after, with a long-value column's maps below the + /// index's: + /// + /// before row2 @3889 pages=[353] row3 @3820 pages=[] row4 @3751 pages=[] + /// after row2 @3958 TOMBSTONE row3 @3889 pages=[] row4 @3820 pages=[] row5 @3751 pages=[355] + /// stale: @3731 = 0x08 (the abandoned append, at 3682) + /// + /// The long-value maps slid up a record width and kept rows 3 and 4; only the index's pointer moved, to + /// the appended row 5. Writing step (1)'s record into the old row's slot instead — which produces the same + /// bytes whenever the recycled row happens to be the LAST one, the only case an ACE-built schema gives — + /// points a slot back up the page as soon as it is not, and no reader can walk that: a row's extent runs + /// to where the previous slot begins. AlterColumnTypeInPlace scans the table through this map + /// immediately afterwards, so the ALTER failed outright on any table that had gained a Memo or OLE + /// column after its index. + /// private int RecycleOwnedMapRow(JetFormatBase format, int usageMapPage, int oldRow, int newRoot) { - const int MapLength = 1 + 4 + 64; int dir = format.DataRowDirectoryOffset; - int rowCount = BinaryPrimitives.ReadUInt16LittleEndian(_channel.ReadPage(usageMapPage).Span.Slice(format.DataRowCountOffset, 2)); + int rowCount = BinaryPrimitives.ReadUInt16LittleEndian( + _channel.ReadPage(usageMapPage).Span.Slice(format.DataRowCountOffset, 2)); int newRow = rowCount; + if (oldRow < 0 || oldRow >= rowCount) + throw new InvalidDataException( + $"Usage-map row {usageMapPage}:{oldRow} does not exist; the page has {rowCount} rows."); - // ACE's first write: append a fresh row and set the new root's bit (this copy is later left stale). + // (1) ACE's first write, kept verbatim: the appended row is where the new root's bit is set, and the + // bytes it leaves behind are part of the file ACE produces. AppendEmptyUsageMapRow(format, usageMapPage, newRow); new UsageMapWriter(_channel).SetBit(newRow, usageMapPage, newRoot, set: true); - // Then move that map into the old row's freed slot and turn the old row into a tombstone; the appended - // slot's bytes are left in place (stale, in free space) — matching ACE's leftover. + // (2) Re-lay the live records. Starting from the page as it stands keeps everything this does not + // write — the abandoned append included — exactly where ACE leaves it. byte[] page = _channel.ReadPage(usageMapPage).Span.ToArray(); - int freshOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(dir + newRow * 2, 2)) & RowPointer.OffsetMask; - int oldOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(dir + oldRow * 2, 2)) & RowPointer.OffsetMask; - int aboveOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(dir + (oldRow - 1) * 2, 2)) & RowPointer.OffsetMask; - - Array.Copy(page, freshOffset, page, oldOffset, MapLength); // move the map into the old slot - BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(dir + newRow * 2, 2), (ushort)oldOffset); - BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(dir + oldRow * 2, 2), - (ushort)(aboveOffset | RowPointer.DeletedFlag | RowPointer.OverflowFlag)); // old row → 0-length tombstone + var holder = new DataPage(); + holder.Read(_channel.ReadPage(usageMapPage), format); + + var records = new byte[rowCount + 1][]; + var flags = new ushort[rowCount + 1]; + for (int i = 0; i <= rowCount; i++) + { + records[i] = i == oldRow ? [] : page.AsSpan(holder.Rows[i].Offset, holder.Rows[i].Length).ToArray(); + flags[i] = (ushort)((i == oldRow || holder.Rows[i].IsDeleted ? RowPointer.DeletedFlag : 0) + | (i == oldRow || holder.Rows[i].HasOverflow ? RowPointer.OverflowFlag : 0)); + } + + int directoryEnd = dir + (rowCount + 1) * 2; + int offset = format.PageSize; + for (int i = 0; i <= rowCount; i++) + { + offset -= records[i].Length; // a 0-length tombstone lands on the previous start + if (offset < directoryEnd) + throw new InvalidOperationException( + $"Usage-map page {usageMapPage} has no room to recycle row {oldRow}: {rowCount} rows already. " + + "The new map belongs on a page of its own."); + records[i].CopyTo(page.AsSpan(offset)); + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(dir + i * 2, 2), + (ushort)(flags[i] | (offset & RowPointer.OffsetMask))); + } BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(format.DataFreeSpaceOffset, 2), - (ushort)(oldOffset - (dir + (rowCount + 1) * 2))); + (ushort)(offset - directoryEnd)); _channel.WritePage(usageMapPage, page); return newRow; } /// Converts a stored value to the CLR type for a new column type (ALTER COLUMN). NULL stays NULL; /// an unconvertible value throws (as ACE's rewrite would). - private static object? ConvertValue(object? value, JetDataType type) + /// Throwing is the intent; the type has to be actionable. The bare Convert.To* calls leaked + /// //, none + /// naming the column and none distinguishable from a bug in the rewrite. + private static object? ConvertValue(object? value, JetDataType type, string columnName) { if (value is null) return null; + try + { + return ConvertCore(value, type); + } + catch (Exception ex) when (ex is InvalidCastException or FormatException or OverflowException + or ArgumentException) + { + throw new InvalidOperationException( + $"Column '{columnName}' cannot be changed to {type}: the existing value " + + $"'{Describe(value)}' ({value.GetType().Name}) cannot be converted to it.", ex); + } + } + + /// Bounded rendering for an error message, so a memo does not paste thousands of characters into + /// one. + private static string Describe(object value) => value switch + { + byte[] bytes => $"{bytes.Length} bytes", + string { Length: > 40 } text => $"{text[..40]}…", + _ => value.ToString() ?? "", + }; + + private static object? ConvertCore(object value, JetDataType type) + { var inv = System.Globalization.CultureInfo.InvariantCulture; return type switch { @@ -2352,7 +2790,7 @@ private int RecycleOwnedMapRow(JetFormatBase format, int usageMapPage, int oldRo JetDataType.Int64 => Convert.ToInt64(value, inv), JetDataType.Single => Convert.ToSingle(value, inv), JetDataType.Double => Convert.ToDouble(value, inv), - JetDataType.Currency or JetDataType.FixedPoint => Convert.ToDecimal(value, inv), + JetDataType.Currency or JetDataType.FixedPoint => JetDecimalConverter.ToDecimal(value, inv), JetDataType.DateTime => value is DateTime d ? d : Convert.ToDateTime(value, inv), JetDataType.Text or JetDataType.Memo => Convert.ToString(value, inv), JetDataType.Guid => value is Guid g ? g : Guid.Parse(value.ToString()!), @@ -2386,8 +2824,11 @@ private static void AppendColumnToParts(TdefParts parts, int colCount, byte[] de /// VariableColumnCount (0x2B stays a high-water mark), or rewrite existing rows — survivors keep /// their stored variable index (§3.4) so old rows still decode (the dropped column's data becomes dead /// bytes). Returns false if the column doesn't exist. Multi-page TDEFs are handled. Throws for a column - /// that backs an index/key (drop that first) or a memo/OLE column (its long-value usage-map entry/pages - /// aren't handled yet). + /// that backs an index/key (drop that first). + /// A memo/OLE column also owns long-value pages through its own usage maps, and ACE retires those as + /// DROP TABLE does (measured by whole-file diff): its §3.3.2 entry leaves the TDEF, its owned and free map + /// records are retired from their holder, and its owned pages go back to the global free map at close. The + /// pages themselves are left as they were, and the other long-value columns keep their entries and records. /// public bool DropColumn(string tableName, string columnName) { @@ -2423,13 +2864,24 @@ public bool DropColumn(string tableName, string columnName) throw new InvalidOperationException( $"Cannot drop column '{columnName}': it is part of an index or key — drop the index/constraint first."); - if (col.Type is JetDataType.Memo or JetDataType.Ole) - throw new NotSupportedException( - $"DROP COLUMN '{columnName}': dropping a memo/OLE (long-value) column is not supported yet."); + // A long-value column's maps, read from the TDEF before it loses them. + var definition = new TableDefinitionPage(); + definition.Read(_channel, table.DefinitionPage); + var maps = new UsageMap(_channel, table); + var owned = new HashSet(); + var retire = new List(); + QueueLongValueMaps(definition, col, maps, owned, retire); TdefParts parts = ParseTdef(table.DefinitionPage); // stitches continuation pages for a multi-page TDEF RemoveColumnFromParts(parts, table.Columns.Count, col.Index, _channel.Format); + RemoveLongValueMapEntry(parts, col.ColumnId); WriteTdef(table.DefinitionPage, parts); + + RetireMapRecords(retire, maps, owned); + var allocator = new PageAllocator(_channel); + foreach (int page in owned) + allocator.Release(page); // reusable only after this handle closes, as ACE holds them + RemoveColumnProperties(table.DefinitionPage, columnName); // drop its DefaultValue/Required from LvProp (ACE does) _catalog.Invalidate(); return true; @@ -2619,9 +3071,9 @@ private void WriteTdef(int tdefPage, TdefParts parts) BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLogicalIndexCountOffset, 4), parts.Logical.Count); BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLengthOffset, 4), defEnd); - // Write across the first page and continuation pages as needed (reusing the existing ones) — handles a + // Write across the first page and continuation pages as needed (fresh ones, the old released) — handles a // definition that shrinks to one page, stays multi-page, or grows past a page (e.g. ADD COLUMN). - WriteDefinition(tdefPage, def, parts.Continuations); + WriteDefinition(tdefPage, def, parts.Continuations, rewrite: true); } /// Marks a row deleted by setting the deleted flag (0x8000) on its slot-directory entry — a @@ -2696,62 +3148,68 @@ private static byte[] BuildOutgoingInfoBlock(int number, int dataOrdinal, byte f /// Writes a definition buffer across the first page and, if it overflows, continuation pages (each /// [0x02][0x01][free:2][next:4] then data). The first page carries the whole definition in its /// coordinate space; each continuation contributes -offset data. - /// Existing continuation pages are reused before allocating new ones. + /// Rewriting an existing definition () is done as ACE does it (verified by + /// whole-file diff, growing and shrinking): the first page is rewritten in place — alone, only the 8-byte + /// reserve past the new end is zeroed and older bytes beyond it are left — and continuation data always goes + /// to freshly allocated pages, while are released untouched. /// - private void WriteDefinition(int firstPage, byte[] def, IReadOnlyList reusePages) + private void WriteDefinition(int firstPage, byte[] def, IReadOnlyList oldContinuations, bool rewrite) { JetFormatBase format = _channel.Format; int ps = format.PageSize; int nextOffset = format.TdefNextPageOffset; + foreach (int old in oldContinuations) + _allocator.Release(old); // reusable only after this handle closes, as ACE holds them + if (def.Length + JetFormatBase.TdefContinuationHeaderSize <= ps) { - var only = new byte[ps]; + byte[] only = rewrite ? _channel.ReadPage(firstPage).Span.ToArray() : new byte[ps]; def.CopyTo(only, 0); + only.AsSpan(def.Length, JetFormatBase.TdefContinuationHeaderSize).Clear(); // the reserve BinaryPrimitives.WriteInt32LittleEndian(only.AsSpan(nextOffset, 4), 0); BinaryPrimitives.WriteUInt16LittleEndian(only.AsSpan(format.TdefFreeSpaceOffset, 2), (ushort)(ps - def.Length - JetFormatBase.TdefContinuationHeaderSize)); _channel.WritePage(firstPage, only); return; } - // Plan the continuation chunks: each holds up to (ps - header) data; the last also leaves the reserve. - // Each continuation carries PageSize - 8 bytes of definition; the LAST one also leaves the 8-byte - // trailing reserve, so it can hold only PageSize - 16. A chunk sized for a middle page that then turns - // out to be the last leaves free space of -8, written as 0xFFF8 — an 8-byte window at every page - // boundary, reachable by ADD COLUMN growing a definition a few bytes at a time. When the remainder - // lands in that window, stop short and let a small tail chunk take the rest. + // The chain holds the definition and then its 8-byte trailing reserve, as ACE lays it out (verified): every + // page is filled before the next begins, and the reserve follows the last definition byte, spilling onto a + // page of its own when it does not fit — so a continuation can hold reserve bytes and no definition. A + // 4,090-byte definition fills page 1 with 4,090 bytes and six of the reserve, and its continuation holds the + // other two, free 4,086. Each page's free space is what it has left once both are placed. int maxMiddle = ps - JetFormatBase.TdefContinuationHeaderSize; - int maxLast = maxMiddle - JetFormatBase.TdefContinuationHeaderSize; - var chunks = new List<(int Offset, int Length)>(); - for (int offset = ps; offset < def.Length;) + int stored = def.Length + JetFormatBase.TdefContinuationHeaderSize; + var chunks = new List<(int Offset, int Length, int Free)>(); + for (int consumed = ps; consumed < stored;) { - int remaining = def.Length - offset; - int length = remaining <= maxLast ? remaining - : remaining > maxMiddle ? maxMiddle - : maxLast; - chunks.Add((offset, length)); - offset += length; + int placed = Math.Min(maxMiddle, stored - consumed); + chunks.Add((consumed, Math.Clamp(def.Length - consumed, 0, placed), maxMiddle - placed)); + consumed += placed; } - int reuse = 0; - int[] pageNumbers = chunks.Select(_ => reuse < reusePages.Count ? reusePages[reuse++] : _allocator.Allocate()).ToArray(); + // ACE allocates the last continuation first (verified: from free pages 354.. a two-page continuation became + // first → 355 → 354, and from the end of a file first → n+1 → n). + var pageNumbers = new int[chunks.Count]; + for (int i = chunks.Count - 1; i >= 0; i--) + pageNumbers[i] = _allocator.Allocate(); var page1 = new byte[ps]; - Array.Copy(def, 0, page1, 0, ps); // page 1 is completely full in a multi-page definition + Array.Copy(def, 0, page1, 0, Math.Min(ps, def.Length)); // page 1 is completely full in a multi-page definition BinaryPrimitives.WriteInt32LittleEndian(page1.AsSpan(nextOffset, 4), pageNumbers[0]); BinaryPrimitives.WriteUInt16LittleEndian(page1.AsSpan(format.TdefFreeSpaceOffset, 2), 0); _channel.WritePage(firstPage, page1); for (int i = 0; i < chunks.Count; i++) { - var (offset, length) = chunks[i]; + var (offset, length, free) = chunks[i]; var page = new byte[ps]; page[0] = (byte)PageType.TableDefinition; page[1] = 0x01; - Array.Copy(def, offset, page, JetFormatBase.TdefContinuationHeaderSize, length); + if (length > 0) // a page holding only the reserve starts past the definition's end + Array.Copy(def, offset, page, JetFormatBase.TdefContinuationHeaderSize, length); int next = i + 1 < pageNumbers.Length ? pageNumbers[i + 1] : 0; BinaryPrimitives.WriteInt32LittleEndian(page.AsSpan(nextOffset, 4), next); - int free = next != 0 ? 0 : ps - JetFormatBase.TdefContinuationHeaderSize - length - JetFormatBase.TdefContinuationHeaderSize; BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(format.TdefFreeSpaceOffset, 2), (ushort)free); _channel.WritePage(pageNumbers[i], page); } @@ -2803,15 +3261,13 @@ private static int ReadInt24(LibRed.IO.PageBuffer buf, int offset) => /// /// Adds an incoming-relationship logical index-info block (§3.6) to a parent table's already-written /// TDEF: it reuses the parent's referenced-key data block (no new data block), links back to the - /// child's outgoing block, and grows the logical-index list by one (kept name-sorted). Single - /// definition page only (throws if the definition spans continuation pages). + /// child's outgoing block, and grows the logical-index list by one (kept name-sorted). The definition is + /// rewritten through , so it may span or spill onto continuation pages. /// private void AddIncomingRelationshipBlock(IncomingRelationship inc) { JetFormatBase format = _channel.Format; - var buf = _channel.ReadPage(inc.ParentPage); - if (buf.ReadInt32(format.TdefNextPageOffset) != 0) - throw new NotSupportedException("Adding a relationship to a multi-page table definition is not supported yet."); + (LibRed.IO.PageBuffer buf, IReadOnlyList existingContinuations) = ReadDefinition(inc.ParentPage); int dataCount = buf.ReadInt32(format.TdefIndexCountOffset); // 0x33 real data blocks int logicalCount = buf.ReadInt32(format.TdefLogicalIndexCountOffset); // 0x2F logical blocks @@ -2853,20 +3309,17 @@ private void AddIncomingRelationshipBlock(IncomingRelationship inc) nameBytes.Insert(k, EncodeName(newName)); int newDefEnd = infoStart + blocks.Count * IndexBlockFormat.InfoBlockSize + nameBytes.Sum(n => n.Length) + lvalRegion.Length; - if (newDefEnd > format.PageSize - JetFormatBase.TdefContinuationHeaderSize) - throw new NotSupportedException("No room in the table definition for another relationship (needs a continuation page)."); - var page = buf.Span.ToArray(); + var def = new byte[newDefEnd]; + buf.Span[..infoStart].CopyTo(def); int w = infoStart; - foreach (byte[] b in blocks) { b.CopyTo(page.AsSpan(w)); w += b.Length; } - foreach (byte[] n in nameBytes) { n.CopyTo(page.AsSpan(w)); w += n.Length; } - lvalRegion.CopyTo(page.AsSpan(w)); - - BinaryPrimitives.WriteInt32LittleEndian(page.AsSpan(format.TdefLogicalIndexCountOffset, 4), logicalCount + 1); - BinaryPrimitives.WriteInt32LittleEndian(page.AsSpan(format.TdefLengthOffset, 4), newDefEnd); - BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(format.TdefFreeSpaceOffset, 2), - (ushort)(format.PageSize - newDefEnd - JetFormatBase.TdefContinuationHeaderSize)); - _channel.WritePage(inc.ParentPage, page); + foreach (byte[] b in blocks) { b.CopyTo(def.AsSpan(w)); w += b.Length; } + foreach (byte[] n in nameBytes) { n.CopyTo(def.AsSpan(w)); w += n.Length; } + lvalRegion.CopyTo(def.AsSpan(w)); + + BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLogicalIndexCountOffset, 4), logicalCount + 1); + BinaryPrimitives.WriteInt32LittleEndian(def.AsSpan(format.TdefLengthOffset, 4), newDefEnd); + WriteDefinition(inc.ParentPage, def, existingContinuations, rewrite: true); } private byte[] BuildIncomingInfoBlock(IncomingRelationship inc) diff --git a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs index f9f9467c8..52b14d888 100644 --- a/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs +++ b/src/LibRed/LibRed.Core/Storage/Types/JetTypeCodec.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Text; +using EntityFrameworkCore.Jet.Data; using LibRed.Catalog; using LibRed.Formats; @@ -256,7 +257,7 @@ internal static byte[] Encode(ColumnDef column, JetDataType type, object value) case JetDataType.DateTimeExtended: // ACE 17 DATETIME2 return EncodeExtendedDateTime(Convert.ToDateTime(value, c)); case JetDataType.Currency: - return Bytes(8, b => BinaryPrimitives.WriteInt64LittleEndian(b, (long)decimal.Round(Convert.ToDecimal(value, c) * 10000m))); + return Bytes(8, b => BinaryPrimitives.WriteInt64LittleEndian(b, (long)decimal.Round(JetDecimalConverter.ToDecimal(value, c) * 10000m))); case JetDataType.Guid: // Coerced, not cast: every other type here accepts what the caller has (AsText, AsBinary, ToOaDate, // Convert.To*), and TableCreator.ConvertValue already parses a string GUID on the ALTER path. A hard @@ -272,9 +273,9 @@ string s when Guid.TryParse(s, out Guid parsed) => parsed, case JetDataType.Text: return EncodeText(column, AsText(value, c)); case JetDataType.Binary: - return EncodeBinary(column, AsBinary(value)); + return EncodeBinary(column, AsBinary(column, value)); case JetDataType.FixedPoint: - return EncodeNumeric(Convert.ToDecimal(value, c), column.Scale); + return EncodeNumeric(column, JetDecimalConverter.ToDecimal(value, c)); // Long values (memo/OLE): store the payload inline after the 12-byte descriptor (memo // text as UTF-16LE, OLE as raw bytes). LongValueReader reads this back via the inline @@ -289,7 +290,7 @@ string s when Guid.TryParse(s, out Guid parsed) => parsed, TryCompressText(column, memo, requireCapableFlag: false) ?? Encoding.Unicode.GetBytes(memo)); } case JetDataType.Ole: - return EncodeInlineLongValue(AsBinary(value)); + return EncodeInlineLongValue(AsBinary(column, value)); default: throw new NotSupportedException($"Encoding {column.Type} is not supported yet."); @@ -407,6 +408,15 @@ private static byte[] EncodeBinary(ColumnDef column, byte[] value) return padded; } + /// 10^, for exponents up to + /// (10^28 is inside decimal's range; 10^29 is not). + private static decimal Pow10(byte exponent) + { + decimal result = 1m; + for (int i = 0; i < exponent; i++) result *= 10m; + return result; + } + private static byte[] Bytes(int length, Action> write) { var b = new byte[length]; @@ -427,11 +437,16 @@ private static byte[] Bytes(int length, Action> write) /// literal: Access has no digitless 0x (it rejects it), which is why /// JetByteArrayTypeMapping emits '' for an empty array. /// - private static byte[] AsBinary(object value) => value switch + /// The fallback names the column rather than hard-casting: a cast raised + /// , which names nothing and is indistinguishable from a bug in the + /// writer. Same fix as the GUID case above. + private static byte[] AsBinary(ColumnDef column, object value) => value switch { byte[] bytes => bytes, string text => Encoding.Unicode.GetBytes(text), - _ => (byte[])value, + _ => throw new NotSupportedException( + $"Cannot store {value.GetType().Name} in {column.Type} column '{column.Name}': a binary column " + + "takes a byte array, or a string (stored as its UTF-16LE bytes)."), }; /// The OLE-automation epoch (1899-12-30), which is also Jet's zero date and the base for @@ -469,11 +484,32 @@ internal static byte[] EncodeInlineLongValue(ReadOnlySpan payload) } /// Inverse of : 17 bytes, sign + 128-bit magnitude (top word 0). - private static byte[] EncodeNumeric(decimal value, byte scale) + /// The largest precision Jet/ACE accepts on a NUMERIC/DECIMAL column. + internal const byte MaxNumericPrecision = 28; + + private static byte[] EncodeNumeric(ColumnDef column, decimal value) { + byte scale = column.Scale; + + // ACE refuses a value wider than the declared precision, on every write path including an ALTER that + // narrows the column. The 17-byte payload cannot enforce it — a 20-digit value fits DECIMAL(18,4) as + // comfortably as a 2-digit one. Tested before scaling, which also stops an out-of-range input + // overflowing the multiply below and reporting OverflowException instead of this. + if (column.Precision is >= 1 and <= MaxNumericPrecision && scale <= column.Precision + && Math.Abs(value) >= Pow10((byte)(column.Precision - scale))) + throw new InvalidOperationException( + $"Value {value} does not fit column '{column.Name}', declared " + + $"DECIMAL({column.Precision},{column.Scale}): it holds at most {column.Precision - scale} " + + $"digits before the decimal point. Access refuses such a value rather than storing it."); + decimal factor = 1m; for (int i = 0; i < scale; i++) factor *= 10m; - decimal magnitude = decimal.Truncate(decimal.Round(Math.Abs(value) * factor, 0)); + + // Truncate toward zero: ACE coerces excess scale rather than refusing it, and truncation matches it in + // every measured case (1.23456 → 1.2345, 1.99999 → 1.9999, -1.23455 → -1.2345). Was decimal.Round(…, 0) + // — ToEven — which differed silently, each engine reading its own answer back happily. + // IndexKeyEncoder.EncodeFixedPoint must quantise identically or keys stop matching their rows. + decimal magnitude = decimal.Truncate(Math.Abs(value) * factor); int[] bits = decimal.GetBits(magnitude); // [lo, mid, hi, flags]; magnitude has scale 0 var result = new byte[17]; diff --git a/src/LibRed/LibRed.Core/Storage/UsageMap.cs b/src/LibRed/LibRed.Core/Storage/UsageMap.cs index 19190ce10..adb0a40b8 100644 --- a/src/LibRed/LibRed.Core/Storage/UsageMap.cs +++ b/src/LibRed/LibRed.Core/Storage/UsageMap.cs @@ -37,6 +37,38 @@ public sealed class UsageMap(PageChannel channel, TableDef table) /// being appended to, so it is the map to consult when looking for somewhere to put a new row. public IEnumerable FreeDataPages() => PagesAt(_channel.Format.TdefFreePagesOffset); + /// The pages recorded by the usage map at an explicit : pointer, rather than one of the TDEF's two fixed-offset maps. A long-value column's + /// owned and free maps are reached this way — their pointers sit in the TDEF keyed by column id, so the + /// pages holding a table's Memo/OLE content are invisible to . + public IEnumerable PagesInMap(int mapRow, int mapPage) => ReadMapAt(mapRow, mapPage); + + /// The dedicated bitmap pages (type 0x05) a reference-form map record at the pointer names, each + /// validated; none for an inline record. + public IReadOnlyList BitmapPagesOf(int mapRow, int mapPage) + { + if (mapPage <= 1 || mapPage >= _channel.PageCount) + throw new InvalidDataException( + $"Usage-map pointer names page {mapPage}, outside the file's 2..{_channel.PageCount - 1} range."); + var holder = new DataPage(); + holder.Read(_channel.ReadPage(mapPage), _channel.Format); + if (mapRow < 0 || mapRow >= holder.RowCount) + throw new InvalidDataException($"Usage-map row {mapPage}:{mapRow} does not exist."); + ReadOnlySpan map = holder.GetRow(mapRow); + if (map.Length == 0 || map[0] != MapTypeReference) return []; + + ValidateReferenceRecord(map); + var pages = new List(); + for (int e = 0; e < ReferenceMapSlots; e++) + { + int bitmapPage = BinaryPrimitives.ReadInt32LittleEndian(map.Slice(1 + e * 4, 4)); + if (bitmapPage == 0) continue; + _ = ReadBitmapPage(bitmapPage); + pages.Add(bitmapPage); + } + return pages; + } + /// The highest-numbered data page the table owns, or -1 when it owns none. /// /// Scans the bitmap backwards rather than enumerating and taking the maximum: @@ -106,11 +138,14 @@ private byte[] ReadMapRecord(int pointerOffset) /// the TDEF. Both maps share the same pointer shape and record format. private IEnumerable PagesAt(int pointerOffset) { - JetFormatBase format = _channel.Format; - PageBuffer tdef = _channel.ReadPage(_table.DefinitionPage); - int mapRow = tdef.ReadByte(pointerOffset); - int mapPage = tdef.ReadInt24(pointerOffset + 1); + return ReadMapAt(tdef.ReadByte(pointerOffset), tdef.ReadInt24(pointerOffset + 1)); + } + + /// Reads the usage-map record at a (row, page) pointer. Shared by the TDEF's own two maps and by + /// the per-column long-value maps, which differ only in where the pointer is stored. + private List ReadMapAt(int mapRow, int mapPage) + { // Both halves of the pointer come out of the TDEF, so both are corruption when wrong. Unchecked, the // page number reached the channel as an out-of-range read and the row number reached GetRow as an // index; the long-value map's equivalent pointer is validated the same way in RowInserter.MapPages. @@ -119,7 +154,9 @@ private IEnumerable PagesAt(int pointerOffset) $"Usage-map pointer names page {mapPage}, outside the file's 2..{_channel.PageCount - 1} range."); var holder = new DataPage(); - holder.Read(_channel.ReadPage(mapPage), format); + holder.Read(_channel.ReadPage(mapPage), _channel.Format); + if (mapRow < 0 || mapRow >= holder.RowCount) + throw new InvalidDataException($"Usage-map row {mapPage}:{mapRow} does not exist."); ReadOnlySpan map = holder.GetRow(mapRow); if (map.Length == 0) @@ -190,10 +227,10 @@ private ReadOnlySpan ReadBitmapPage(int pageNumber) /// in RowInserter already range-checks; this is the same check. private void AppendSetBits(List pages, ReadOnlySpan bitmap, int basePage) { - // Read the bound ONCE. Outside a transaction PageChannel.PageCount is a file-length syscall (inside one - // it is a cached field), and this loop runs per set bit on every insert — so testing it per bit made a - // non-transactional insert ~1.9x slower while leaving the transactional path untouched, which is what - // made the cost so easy to miss. Nothing in the loop writes, so the count cannot move under it. + // Read the bound ONCE: this loop runs per set bit on every insert. PageChannel.PageCount used to be a + // file-length syscall outside a transaction, which made a per-bit test cost a non-transactional insert + // ~1.9x; it is a cached field now, but one read is still all the loop needs. Nothing in the loop writes, + // so the count cannot move under it. int pageCount = _channel.PageCount; for (int i = 0; i < bitmap.Length; i++) diff --git a/src/LibRed/LibRed.Core/Storage/ViewCreator.cs b/src/LibRed/LibRed.Core/Storage/ViewCreator.cs index afa72a386..be6b09454 100644 --- a/src/LibRed/LibRed.Core/Storage/ViewCreator.cs +++ b/src/LibRed/LibRed.Core/Storage/ViewCreator.cs @@ -25,13 +25,17 @@ public sealed class ViewCreator(PageChannel channel, JetCatalog catalog) private const int QueryOwnerAcm = 0xF00FE; // 983294 private const int QueryAdminAcm = 0xFFEFF; // 1048319 + // A relationship object's MSysACEs rows (verified vs ACE): owner 0xF00FE as a query's, admin 0xFFFFF. + private const int RelationshipOwnerAcm = 0xF00FE; // 983294 + private const int RelationshipAdminAcm = 0xFFFFF; // 1048575 + private readonly PageChannel _channel = channel; private readonly JetCatalog _catalog = catalog; public void Create(string name, ViewSpec spec) { - int objectId = AllocateObject(name, ViewFlags); + int objectId = AllocateQueryObject(name, ViewFlags); AddQueryRows(objectId, spec); } @@ -39,45 +43,66 @@ public void Create(string name, ViewSpec spec) public void CreateAction(string name, ActionQuerySpec spec) { int flags = spec.Kind == ActionQueryKind.DataDefinition ? DataDefinitionFlags : AppendFlags; - int objectId = AllocateObject(name, flags); + int objectId = AllocateQueryObject(name, flags); AddActionRows(objectId, spec); } - /// Reserves the next free query object id, checks the name is unique, and writes the MSysObjects - /// row with the given (which distinguish view / append / data-definition). - private int AllocateObject(string name, int flags) + /// + /// Records a relationship as ACE does (verified): a type-8 MSysObjects object in the Relationships + /// container, named after it, flags 0, with the next high-bit id — the sequence queries draw from, so the two + /// interleave, and a dropped one's id is taken again — and its two MSysACEs rows. Refuses a name + /// another relationship has, as ACE does; a table or query may share it. + /// + public void CreateRelationshipObject(string name) => + AllocateObject(name, CatalogFormat.ObjectTypeRelationship, CatalogFormat.RelationshipContainerParentId, flags: 0, + RelationshipOwnerAcm, RelationshipAdminAcm); + + private int AllocateQueryObject(string name, int flags) => + AllocateObject(name, StoredQueryFormat.ObjectTypeQuery, CatalogFormat.ObjectContainerParentId, flags, + QueryOwnerAcm, QueryAdminAcm); + + /// Reserves the next free high-bit object id, checks the name is free, and writes + /// the MSysObjects row and its two MSysACEs rows. For a query the distinguish view / + /// append / data-definition. + private int AllocateObject(string name, short type, int parentId, int flags, int ownerAcm, int adminAcm) { TableDef msysObjects = _catalog.FindTable("MSysObjects") ?? throw new InvalidOperationException("MSysObjects catalog table was not found."); int idIndex = ColumnIndex(msysObjects, "Id"); int nameIndex = ColumnIndex(msysObjects, "Name"); + int parentIndex = ColumnIndex(msysObjects, "ParentId"); - // A query's name must be unique among all objects (it also cannot equal an existing table name); - // find the next free negative id (queries increment from 0x80000000) in one scan. + // A query's name must be unique among all objects (it also cannot equal an existing table name); a + // relationship's only among the relationships, as ACE has it. Find the next free negative id (they + // increment from 0x80000000) in the same scan. + bool relationship = parentId == CatalogFormat.RelationshipContainerParentId; int nextId = unchecked((int)0x80000000); foreach (object?[] row in new Table(_channel, msysObjects).Rows()) { - if (string.Equals(row[nameIndex] as string, name, StringComparison.OrdinalIgnoreCase)) - throw new SchemaObjectExistsException($"An object named '{name}' already exists.", name); + if ((!relationship || row[parentIndex] is int parent && parent == parentId) + && string.Equals(row[nameIndex] as string, name, StringComparison.OrdinalIgnoreCase)) + throw new SchemaObjectExistsException(relationship + ? $"There is already a relationship named '{name}' in the current database." + : $"An object named '{name}' already exists.", name); if (row[idIndex] is int id && id < 0 && id >= nextId) nextId = id + 1; } - AddObjectRow(msysObjects, name, nextId, flags); - AddPermissionRows(nextId); + AddObjectRow(msysObjects, name, nextId, type, parentId, flags); + AddPermissionRows(nextId, ownerAcm, adminAcm); return nextId; } /// - /// Adds the two MSysACEs permission rows Access writes for a new query/view object — owner (0x690C) at - /// ACM 0xF00FE and admin/users (0x680C) at ACM 0xFFEFF — maintaining the ObjectId index so Access's - /// security check finds them. Without these Access warns about permissions when opening the query. + /// Adds the two MSysACEs permission rows Access writes for a new query/view or relationship object — owner + /// (0x690C) and admin/users (0x680C) — maintaining the ObjectId index so Access's security check finds them. + /// Without these Access warns about permissions when opening a query. /// - private void AddPermissionRows(int objectId) + private void AddPermissionRows(int objectId, int ownerAcm, int adminAcm) { TableDef msysAces = _catalog.FindTable("MSysACEs") ?? throw new InvalidOperationException("MSysACEs catalog table was not found."); - foreach ((byte[] sid, int acm) in new[] { (DefaultOwner, QueryOwnerAcm), (AdminSid, QueryAdminAcm) }) + foreach ((byte[] sid, int acm) in new[] { (DefaultOwner, ownerAcm), (AdminSid, adminAcm) }) { var values = new object?[msysAces.Columns.Count]; SetByName(msysAces, values, "ACM", acm); @@ -88,13 +113,13 @@ private void AddPermissionRows(int objectId) } } - private void AddObjectRow(TableDef msysObjects, string name, int objectId, int flags) + private void AddObjectRow(TableDef msysObjects, string name, int objectId, short type, int parentId, int flags) { DateTime now = DateTime.Now; var values = new object?[msysObjects.Columns.Count]; SetByName(msysObjects, values, "Id", objectId); - SetByName(msysObjects, values, "ParentId", CatalogFormat.ObjectContainerParentId); - SetByName(msysObjects, values, "Type", StoredQueryFormat.ObjectTypeQuery); + SetByName(msysObjects, values, "ParentId", parentId); + SetByName(msysObjects, values, "Type", type); SetByName(msysObjects, values, "Name", name); SetByName(msysObjects, values, "Flags", flags); SetByName(msysObjects, values, "Owner", DefaultOwner); diff --git a/src/LibRed/LibRed.EFCore/Extensions/LibRedServiceCollectionExtensions.cs b/src/LibRed/LibRed.EFCore/Extensions/LibRedServiceCollectionExtensions.cs index 8744b6e37..c84eb3a26 100644 --- a/src/LibRed/LibRed.EFCore/Extensions/LibRedServiceCollectionExtensions.cs +++ b/src/LibRed/LibRed.EFCore/Extensions/LibRedServiceCollectionExtensions.cs @@ -51,11 +51,11 @@ public static IServiceCollection AddEntityFrameworkLibRed(this IServiceCollectio .TryAdd() .TryAdd(p => p.GetRequiredService()) .TryAdd() - .TryAdd() + .TryAdd() .TryAdd() .TryAdd() .TryAdd() - .TryAdd() + .TryAdd() .TryAdd() .TryAdd() .TryAdd() diff --git a/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMathTranslator.cs b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMathTranslator.cs new file mode 100644 index 000000000..6c2e50776 --- /dev/null +++ b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMathTranslator.cs @@ -0,0 +1,197 @@ +using EntityFrameworkCore.Jet.Query; +using EntityFrameworkCore.Jet.Utilities; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using ExpressionExtensions = Microsoft.EntityFrameworkCore.Query.ExpressionExtensions; + +namespace EntityFrameworkCore.LibRed.Query.Internal; + +/// +/// This API supports the Entity Framework Core infrastructure and is not intended to be used +/// directly from your code. This API may change or be removed in future releases. +/// +public class LibRedMathTranslator(ISqlExpressionFactory sqlExpressionFactory) : IMethodCallTranslator +{ + private readonly JetSqlExpressionFactory _sqlExpressionFactory = (JetSqlExpressionFactory)sqlExpressionFactory; + + private static readonly Dictionary _supportedMethodTranslationsDirect = new() + { + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(decimal)])!, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(double)])!, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(float)])!, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(int)]) !, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(long)]) !, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(sbyte)]) !, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Abs), [typeof(short)]) !, "ABS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Pow), [typeof(double), typeof(double)]) !, "POWER"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Exp), [typeof(double)]) !, "EXP"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Log), [typeof(double)])!, "LN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Log10), [typeof(double)])!, "LOG10"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sqrt), [typeof(double)])!, "SQRT"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Floor), [typeof(decimal)])!, "FLOOR"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Floor), [typeof(double)])!, "FLOOR"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Ceiling), [typeof(decimal)])!, "CEILING"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Ceiling), [typeof(double)])!, "CEILING"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Acos), [typeof(double)])!, "ACOS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Asin), [typeof(double)])!, "ASIN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Atan), [typeof(double)])!, "ATAN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Atan2), [typeof(double), typeof(double)])!, "ATAN2"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Cos), [typeof(double)])!, "COS"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sin), [typeof(double)])!, "SIN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Tan), [typeof(double)])!, "TAN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Cosh), [typeof(double)])!, "COSH"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sinh), [typeof(double)])!, "SINH"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Tanh), [typeof(double)])!, "TANH"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(decimal)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(double)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(float)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(int)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(long)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(sbyte)])!, "SIGN"}, + {typeof(Math).GetRuntimeMethod(nameof(Math.Sign), [typeof(short)])!, "SIGN"}, + {typeof(double).GetRuntimeMethod(nameof(double.DegreesToRadians), [typeof(double)])!, "RADIANS"}, + {typeof(double).GetRuntimeMethod(nameof(double.RadiansToDegrees), [typeof(double)])!, "DEGREES"}, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Abs), [typeof(float)])!, "ABS" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Pow), [typeof(float), typeof(float)])!, "POWER" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Exp), [typeof(float)])!, "EXP" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Log), [typeof(float)])!, "LN" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Log10), [typeof(float)])!, "LOG10" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Sqrt), [typeof(float)])!, "SQRT" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Floor), [typeof(float)])!, "FLOOR" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Ceiling), [typeof(float)])!, "CEILING" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Acos), [typeof(float)])!, "ACOS" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Asin), [typeof(float)])!, "ASIN" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Atan), [typeof(float)])!, "ATAN" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Atan2), [typeof(float), typeof(float)])!, "ATAN2" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Cos), [typeof(float)])!, "COS" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Sin), [typeof(float)])!, "SIN" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Tan), [typeof(float)])!, "TAN" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Cosh), [typeof(float)])!, "COSH" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Sinh), [typeof(float)])!, "SINH" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Tanh), [typeof(float)])!, "TANH" }, + { typeof(MathF).GetRuntimeMethod(nameof(MathF.Sign), [typeof(float)])!, "SIGN" }, + { typeof(float).GetRuntimeMethod(nameof(float.DegreesToRadians), [typeof(float)])!, "RADIANS" }, + { typeof(float).GetRuntimeMethod(nameof(float.RadiansToDegrees), [typeof(float)])!, "DEGREES" } + }; + + private static readonly IEnumerable _logMethodInfos = + [ + typeof(Math).GetRuntimeMethod(nameof(Math.Log), [typeof(double), typeof(double)])!, + typeof(MathF).GetRuntimeMethod(nameof(MathF.Log), [typeof(float), typeof(float)])! + ]; + + private static readonly IEnumerable _truncateMethodInfos = + [ + typeof(Math).GetRuntimeMethod(nameof(Math.Truncate), [typeof(decimal)])!, + typeof(Math).GetRuntimeMethod(nameof(Math.Truncate), [typeof(double)])!, + typeof(MathF).GetRuntimeMethod(nameof(MathF.Truncate), [typeof(float)])! + ]; + + private static readonly IEnumerable _roundMethodInfos = + [ + typeof(Math).GetRuntimeMethod(nameof(Math.Round), [typeof(decimal)])!, + typeof(Math).GetRuntimeMethod(nameof(Math.Round), [typeof(double)])!, + typeof(Math).GetRuntimeMethod(nameof(Math.Round), [typeof(decimal), typeof(int)])!, + typeof(Math).GetRuntimeMethod(nameof(Math.Round), [typeof(double), typeof(int)])!, + typeof(MathF).GetRuntimeMethod(nameof(MathF.Round), [typeof(float)])!, + typeof(MathF).GetRuntimeMethod(nameof(MathF.Round), [typeof(float), typeof(int)])! + ]; + + public SqlExpression? Translate(SqlExpression? instance, MethodInfo method, IReadOnlyList arguments, IDiagnosticsLogger logger) + { + Check.NotNull(method, nameof(method)); + + if (_supportedMethodTranslationsDirect.TryGetValue(method, out var sqlFunctionName)) + { + var typeMapping = arguments.Count == 1 + ? ExpressionExtensions.InferTypeMapping(arguments[0]) + : ExpressionExtensions.InferTypeMapping(arguments[0], arguments[1]); + + var newArguments = new SqlExpression[arguments.Count]; + newArguments[0] = _sqlExpressionFactory.ApplyTypeMapping(arguments[0], typeMapping); + + if (arguments.Count == 2) + { + newArguments[1] = _sqlExpressionFactory.ApplyTypeMapping(arguments[1], typeMapping); + } + + return _sqlExpressionFactory.Function( + sqlFunctionName, + newArguments, + nullable: true, + argumentsPropagateNullability: newArguments.Select(_ => true).ToArray(), + method.ReturnType, + sqlFunctionName == "SIGN" ? null : typeMapping); + } + + if (_logMethodInfos.Contains(method)) + { + // Math.Log(x, n) is LOG(n, x): the engine's LOG takes the base first + var typeMapping = ExpressionExtensions.InferTypeMapping(arguments[0], arguments[1]); + + return _sqlExpressionFactory.Function( + "LOG", + [ + _sqlExpressionFactory.ApplyTypeMapping(arguments[1], typeMapping), + _sqlExpressionFactory.ApplyTypeMapping(arguments[0], typeMapping) + ], + nullable: true, + argumentsPropagateNullability: [true, true], + method.ReturnType, + typeMapping); + } + + if (_truncateMethodInfos.Contains(method)) + { + var argument = arguments[0]; + // C# has Round over decimal/double/float only so our argument will be one of those types (compiler puts convert node) + // In database result will be same type except for float which returns double which we need to cast back to float. + var resultType = argument.Type; + if (resultType == typeof(float)) + { + resultType = typeof(double); + } + var result = (SqlExpression)_sqlExpressionFactory.Function( + "FIX", + [argument], + nullable: true, + argumentsPropagateNullability: [true], + resultType); + + if (argument.Type == typeof(float)) + { + result = _sqlExpressionFactory.Convert(result, typeof(float)); + } + + return _sqlExpressionFactory.ApplyTypeMapping(result, argument.TypeMapping); + } + + if (_roundMethodInfos.Contains(method)) + { + var argument = arguments[0]; + var digits = arguments.Count == 2 ? arguments[1] : _sqlExpressionFactory.Constant(0); + // C# has Round over decimal/double/float only so our argument will be one of those types (compiler puts convert node) + // In database result will be same type except for float which returns double which we need to cast back to float. + var resultType = argument.Type; + if (resultType == typeof(float)) + { + resultType = typeof(double); + } + + var result = (SqlExpression)_sqlExpressionFactory.Function( + "ROUND", + [argument, digits], + nullable: false, + argumentsPropagateNullability: [true, false], + resultType); + + if (argument.Type == typeof(float)) + { + result = _sqlExpressionFactory.Convert(result, typeof(float)); + } + + return _sqlExpressionFactory.ApplyTypeMapping(result, argument.TypeMapping); + } + + return null; + } +} diff --git a/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMethodCallTranslatorProvider.cs b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMethodCallTranslatorProvider.cs new file mode 100644 index 000000000..cf244d496 --- /dev/null +++ b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedMethodCallTranslatorProvider.cs @@ -0,0 +1,45 @@ +using EntityFrameworkCore.Jet.Query; +using EntityFrameworkCore.Jet.Query.ExpressionTranslators.Internal; +using EntityFrameworkCore.LibRed.Infrastructure; +using EntityFrameworkCore.LibRed.Infrastructure.Internal; + +namespace EntityFrameworkCore.LibRed.Query.Internal; + +/// +/// This API supports the Entity Framework Core infrastructure and is not intended to be used +/// directly from your code. This API may change or be removed in future releases. +/// +public class LibRedMethodCallTranslatorProvider : RelationalMethodCallTranslatorProvider +{ + /// + /// This API supports the Entity Framework Core infrastructure and is not intended to be used + /// directly from your code. This API may change or be removed in future releases. + /// + public LibRedMethodCallTranslatorProvider( + RelationalMethodCallTranslatorProviderDependencies dependencies, + ILibRedOptions options) + : base(dependencies) + { + var sqlExpressionFactory = (JetSqlExpressionFactory)dependencies.SqlExpressionFactory; + + // ReSharper disable once VirtualMemberCallInConstructor + AddTranslators( + [ + new JetByteArrayMethodTranslator(sqlExpressionFactory), + new JetConvertTranslator(sqlExpressionFactory), + new JetDateDiffFunctionsTranslator(sqlExpressionFactory), + new JetDateOnlyMethodTranslator(sqlExpressionFactory), + new JetDateTimeMethodTranslator(sqlExpressionFactory), + new JetIsDateFunctionTranslator(sqlExpressionFactory), + options.SqlMode == LibRedSqlMode.Compatible + ? new JetMathTranslator(sqlExpressionFactory) + : new LibRedMathTranslator(sqlExpressionFactory), + new JetNewGuidTranslator(sqlExpressionFactory), + new JetObjectToStringTranslator(sqlExpressionFactory), + new JetParseTranslator(sqlExpressionFactory), + new JetStringMethodTranslator(sqlExpressionFactory), + new JetRandomTranslator(sqlExpressionFactory), + new JetTimeOnlyMethodTranslator(sqlExpressionFactory) + ]); + } +} diff --git a/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitor.cs b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitor.cs new file mode 100644 index 000000000..120ee5c50 --- /dev/null +++ b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitor.cs @@ -0,0 +1,617 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using System.Text; +using EntityFrameworkCore.Jet; +using EntityFrameworkCore.Jet.Internal; +using ExpressionExtensions = Microsoft.EntityFrameworkCore.Query.ExpressionExtensions; + +namespace EntityFrameworkCore.LibRed.Query.Internal; + +/// +/// LibRed's own SQL translating expression visitor, used in +/// mode. It is a copy of the shared JetSqlTranslatingExpressionVisitor — which the compatible mode keeps +/// using, because ACE has to run what that mode generates — and differs where LibRed's engine goes beyond ACE: +/// Math.Max/Math.Min, and a Max()/Min() over an inline collection, become the +/// standard GREATEST/LEAST rather than a nest of CASE comparisons. +/// +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class LibRedSqlTranslatingExpressionVisitor( + RelationalSqlTranslatingExpressionVisitorDependencies dependencies, + QueryCompilationContext queryCompilationContext, + QueryableMethodTranslatingExpressionVisitor queryableMethodTranslatingExpressionVisitor) : RelationalSqlTranslatingExpressionVisitor(dependencies, queryCompilationContext, queryableMethodTranslatingExpressionVisitor) +{ + private readonly QueryCompilationContext _queryCompilationContext = queryCompilationContext; + private readonly ISqlExpressionFactory _sqlExpressionFactory = dependencies.SqlExpressionFactory; + + private static readonly HashSet DateTimeDataTypes + = + [ + "time", + "date", + "datetime", + "datetime2", + "datetimeoffset" + ]; + + private static readonly HashSet DateTimeClrTypes + = + [ + typeof(TimeOnly), + typeof(DateOnly), + typeof(TimeSpan), + typeof(DateTime), + typeof(DateTimeOffset) + ]; + + private static readonly HashSet ArithmeticOperatorTypes + = + [ + ExpressionType.Add, + ExpressionType.Subtract, + ExpressionType.Multiply, + ExpressionType.Divide, + ExpressionType.Modulo + ]; + + private static readonly MethodInfo StringStartsWithMethodInfoString + = typeof(string).GetRuntimeMethod(nameof(string.StartsWith), [typeof(string)])!; + + private static readonly MethodInfo StringStartsWithMethodInfoChar + = typeof(string).GetRuntimeMethod(nameof(string.StartsWith), [typeof(char)])!; + + private static readonly MethodInfo StringEndsWithMethodInfoString + = typeof(string).GetRuntimeMethod(nameof(string.EndsWith), [typeof(string)])!; + + private static readonly MethodInfo StringEndsWithMethodInfoChar + = typeof(string).GetRuntimeMethod(nameof(string.EndsWith), [typeof(char)])!; + + private static readonly MethodInfo StringContainsMethodInfoString + = typeof(string).GetRuntimeMethod(nameof(string.Contains), [typeof(string)])!; + + private static readonly MethodInfo StringContainsMethodInfoChar + = typeof(string).GetRuntimeMethod(nameof(string.Contains), [typeof(char)])!; + + private static readonly MethodInfo EscapeLikePatternParameterMethod = + typeof(LibRedSqlTranslatingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ConstructLikePatternParameter))!; + + private static readonly MethodInfo StringJoinMethodInfo + = typeof(string).GetRuntimeMethod(nameof(string.Join), [typeof(string), typeof(string[])])!; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override Expression VisitBinary(BinaryExpression binaryExpression) + { + if (binaryExpression.NodeType == ExpressionType.ArrayIndex + && binaryExpression.Left.Type == typeof(byte[])) + { + return TranslateByteArrayElementAccess( + binaryExpression.Left, + binaryExpression.Right, + binaryExpression.Type); + } + + var visitedExpression = base.VisitBinary(binaryExpression); + + if (visitedExpression is SqlBinaryExpression be) + { + if (be.OperatorType is ExpressionType.And + or ExpressionType.Or + or ExpressionType.ExclusiveOr) + { + var left = CoerceIfNeeded(be.Left); + var right = CoerceIfNeeded(be.Right); + + if (!ReferenceEquals(left, be.Left) || !ReferenceEquals(right, be.Right)) + { + visitedExpression = new SqlBinaryExpression( + be.OperatorType, + left, + right, + be.Type, + be.TypeMapping); + } + } + } + + if (visitedExpression is SqlBinaryExpression sqlBinaryExpression + && ArithmeticOperatorTypes.Contains(sqlBinaryExpression.OperatorType)) + { + var inferredProviderType = GetProviderType(sqlBinaryExpression.Left) ?? GetProviderType(sqlBinaryExpression.Right); + if (inferredProviderType != null) + { + if (DateTimeDataTypes.Contains(inferredProviderType)) + { + return QueryCompilationContext.NotTranslatedExpression; + } + } + else + { + var leftType = sqlBinaryExpression.Left.Type; + var rightType = sqlBinaryExpression.Right.Type; + if (DateTimeClrTypes.Contains(leftType) + || DateTimeClrTypes.Contains(rightType)) + { + return QueryCompilationContext.NotTranslatedExpression; + } + } + } + + return visitedExpression; + } + + private SqlExpression CoerceIfNeeded(SqlExpression operand) + { + // Skip if already a LONG (Jet) or if already a CLNG(...) call + var storeType = operand.TypeMapping?.StoreType; + if (storeType is "long" or "integer" or "int") return operand; + var clr = operand.Type; + if (clr == typeof(byte) || clr == typeof(short)) + { + return _sqlExpressionFactory.Function( + "CLNG", + [operand], + true, + argumentsPropagateNullability: [true], + typeof(long), + IntTypeMapping.Default); + } + return operand; + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override Expression VisitUnary(UnaryExpression unaryExpression) + { + if (unaryExpression.NodeType == ExpressionType.ArrayLength + && unaryExpression.Operand.Type == typeof(byte[])) + { + if (!(base.Visit(unaryExpression.Operand) is SqlExpression sqlExpression)) + { + return QueryCompilationContext.NotTranslatedExpression; + } + throw new InvalidOperationException(JetStrings.ByteArrayLength); + } + + return base.VisitUnary(unaryExpression); + } + + protected override Expression VisitExtension(Expression extensionExpression) + { + var result = base.VisitExtension(extensionExpression); + if (extensionExpression is ShapedQueryExpression shapedQueryExpression) + { + var shaperExpression = shapedQueryExpression.ShaperExpression; + if (shapedQueryExpression.ResultCardinality == ResultCardinality.SingleOrDefault + && !shaperExpression.Type.IsNullableType() && result is SqlFunctionExpression { Name: "COALESCE"} sqlFunctionExpression) + { + if (sqlFunctionExpression.Arguments?[1] is SqlConstantExpression { Value: DateTime { Ticks: 0 } }) + { + var newconst = new SqlConstantExpression(new DateTime(100, 1, 1), sqlFunctionExpression.Arguments[1].TypeMapping); + return _sqlExpressionFactory.Coalesce(sqlFunctionExpression.Arguments[0], + (SqlExpression)Visit(newconst)); + } + } + } + return result; + } + + protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression) + { + var method = methodCallExpression.Method; + + if (method.IsGenericMethod + && method.GetGenericMethodDefinition() == EnumerableMethods.ElementAt + && methodCallExpression.Arguments[0].Type == typeof(byte[])) + { + return TranslateByteArrayElementAccess( + methodCallExpression.Arguments[0], + methodCallExpression.Arguments[1], + methodCallExpression.Type); + } + + if ((method == StringStartsWithMethodInfoString || method == StringStartsWithMethodInfoChar) + && TryTranslateStartsEndsWithContains( + methodCallExpression.Object!, methodCallExpression.Arguments[0], StartsEndsWithContains.StartsWith, out var translation1)) + { + return translation1; + } + + if ((method == StringEndsWithMethodInfoString || method == StringEndsWithMethodInfoChar) + && TryTranslateStartsEndsWithContains( + methodCallExpression.Object!, methodCallExpression.Arguments[0], StartsEndsWithContains.EndsWith, out var translation2)) + { + return translation2; + } + + if ((method == StringContainsMethodInfoString || method == StringContainsMethodInfoChar) + && TryTranslateStartsEndsWithContains( + methodCallExpression.Object!, methodCallExpression.Arguments[0], StartsEndsWithContains.Contains, out var translation3)) + { + return translation3; + } + + if (method == StringJoinMethodInfo + && methodCallExpression.Arguments[1] is NewArrayExpression newArrayExpression + && Visit(methodCallExpression.Arguments[0]) is SqlExpression separator) + { + var stringTypeMapping = separator.TypeMapping; + + SqlExpression? result = null; + + foreach (var expression in newArrayExpression.Expressions) + { + if (Visit(expression) is not SqlExpression translated) + { + return QueryCompilationContext.NotTranslatedExpression; + } + + translated = _sqlExpressionFactory.Coalesce( + _sqlExpressionFactory.ApplyTypeMapping(translated, stringTypeMapping), + _sqlExpressionFactory.Constant(string.Empty, stringTypeMapping)); + + result = result is null + ? translated + : _sqlExpressionFactory.Add( + _sqlExpressionFactory.Add(result, separator), + translated); + } + + return result ?? _sqlExpressionFactory.Constant(string.Empty, stringTypeMapping); + } + + return base.VisitMethodCall(methodCallExpression); + + bool TryTranslateStartsEndsWithContains( + Expression instance, + Expression pattern, + StartsEndsWithContains methodType, + [NotNullWhen(true)] out SqlExpression? translation) + { + if (Visit(instance) is not SqlExpression translatedInstance + || Visit(pattern) is not SqlExpression translatedPattern) + { + translation = null; + return false; + } + + var stringTypeMapping = ExpressionExtensions.InferTypeMapping(translatedInstance, translatedPattern); + + translatedInstance = _sqlExpressionFactory.ApplyTypeMapping(translatedInstance, stringTypeMapping); + translatedPattern = _sqlExpressionFactory.ApplyTypeMapping(translatedPattern, stringTypeMapping); + + switch (translatedPattern) + { + case SqlConstantExpression patternConstant: + { + // The pattern is constant. Aside from null and empty string, we bracket every wildcard character (see + // IsLikeWildChar) and send a simple LIKE + translation = patternConstant.Value switch + { + null => _sqlExpressionFactory.Like(translatedInstance, _sqlExpressionFactory.Constant(null,typeof(string), stringTypeMapping)), + + // In .NET, all strings start with/end with/contain the empty string, but SQL LIKE return false for empty patterns. + // Return % which always matches instead. + // Note that we don't just return a true constant, since null strings shouldn't match even an empty string + // (but SqlNullabilityProcess will convert this to a true constant if the instance is non-nullable) + "" => _sqlExpressionFactory.Like(translatedInstance, _sqlExpressionFactory.Constant("%")), + + string s => s.Any(IsLikeWildChar) + ? _sqlExpressionFactory.Like( + translatedInstance, + _sqlExpressionFactory.Constant( + methodType switch + { + StartsEndsWithContains.StartsWith => EscapeLikePattern(s) + '%', + StartsEndsWithContains.EndsWith => '%' + EscapeLikePattern(s), + StartsEndsWithContains.Contains => $"%{EscapeLikePattern(s)}%", + + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + })) + : _sqlExpressionFactory.Like( + translatedInstance, + _sqlExpressionFactory.Constant( + methodType switch + { + StartsEndsWithContains.StartsWith => s + '%', + StartsEndsWithContains.EndsWith => '%' + s, + StartsEndsWithContains.Contains => $"%{s}%", + + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + })), + + char s when !IsLikeWildChar(s) + => _sqlExpressionFactory.Like( + translatedInstance, + _sqlExpressionFactory.Constant( + methodType switch + { + StartsEndsWithContains.StartsWith => s + "%", + StartsEndsWithContains.EndsWith => "%" + s, + StartsEndsWithContains.Contains => $"%{s}%", + + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + })), + + // LibRed, like Jet, has no ESCAPE clause, so a wildcard character is bracketed as in a string pattern. + char s => _sqlExpressionFactory.Like( + translatedInstance, + _sqlExpressionFactory.Constant( + methodType switch + { + StartsEndsWithContains.StartsWith => EscapeLikePattern(s.ToString()) + "%", + StartsEndsWithContains.EndsWith => "%" + EscapeLikePattern(s.ToString()), + StartsEndsWithContains.Contains => $"%{EscapeLikePattern(s.ToString())}%", + + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + })), + + _ => throw new UnreachableException() + }; + + return true; + } + + case SqlParameterExpression patternParameter: + { + // The pattern is a parameter, register a runtime parameter that will contain the rewritten LIKE pattern, where + // all special characters have been escaped. + var lambda = Expression.Lambda( + Expression.Call( + EscapeLikePatternParameterMethod, + QueryCompilationContext.QueryContextParameter, + Expression.Constant(patternParameter.Name), + Expression.Constant(methodType)), + QueryCompilationContext.QueryContextParameter); + + var escapedPatternParameter = + _queryCompilationContext.RegisterRuntimeParameter( + $"{patternParameter.Name}_{methodType.ToString().ToLower(CultureInfo.InvariantCulture)}", lambda); + + translation = _sqlExpressionFactory.Like( + translatedInstance, + new SqlParameterExpression(escapedPatternParameter.Name!, escapedPatternParameter.Type, stringTypeMapping)); + + return true; + } + + default: + // The pattern is a column or a complex expression; the possible special characters in the pattern cannot be escaped, + // preventing us from translating to LIKE. + translation = methodType switch + { + // For StartsWith/EndsWith, use LEFT or RIGHT instead to extract substring and compare: + // WHERE instance IS NOT NULL AND pattern IS NOT NULL AND LEFT(instance, LEN(pattern)) = pattern + // This is less efficient than LIKE (i.e. StartsWith does an index scan instead of seek), but we have no choice. + // Note that we compensate for the case where both the instance and the pattern are null (null.StartsWith(null)); a + // simple equality would yield true in that case, but we want false. We technically + StartsEndsWithContains.StartsWith or StartsEndsWithContains.EndsWith + => _sqlExpressionFactory.AndAlso( + _sqlExpressionFactory.IsNotNull(translatedInstance), + _sqlExpressionFactory.AndAlso( + _sqlExpressionFactory.IsNotNull(translatedPattern), + _sqlExpressionFactory.Equal( + _sqlExpressionFactory.Function( + methodType is StartsEndsWithContains.StartsWith ? "LEFT" : "RIGHT", + [ + translatedInstance, + _sqlExpressionFactory.Coalesce( + _sqlExpressionFactory.Function( + "LEN", + [translatedPattern], + nullable: true, + argumentsPropagateNullability: [false], + typeof(int)), + _sqlExpressionFactory.Constant(0) + ) + ], + nullable: true, + argumentsPropagateNullability: [true, false], + typeof(string), + stringTypeMapping), + translatedPattern))), + + // For Contains, just use INSTR and check if the result is greater than 0. + // Add a check to return null when the pattern is an empty string (and the string isn't null) + StartsEndsWithContains.Contains + => _sqlExpressionFactory.AndAlso( + _sqlExpressionFactory.IsNotNull(translatedInstance), + _sqlExpressionFactory.AndAlso( + _sqlExpressionFactory.IsNotNull(translatedPattern), + _sqlExpressionFactory.OrElse( + _sqlExpressionFactory.GreaterThan( + _sqlExpressionFactory.Function( + "INSTR", + [_sqlExpressionFactory.Constant(1), translatedInstance, translatedPattern, _sqlExpressionFactory.Constant(1)], + nullable: true, + argumentsPropagateNullability: [false, true, true, false], + typeof(int)), + _sqlExpressionFactory.Constant(0)), + _sqlExpressionFactory.Like( + translatedPattern, + _sqlExpressionFactory.Constant(string.Empty, stringTypeMapping))))), + + _ => throw new UnreachableException() + }; + + return true; + } + } + } + + public static string? ConstructLikePatternParameter( + QueryContext queryContext, + string baseParameterName, + StartsEndsWithContains methodType) + => queryContext.Parameters[baseParameterName] switch + { + null => null, + + // In .NET, all strings start/end with the empty string, but SQL LIKE return false for empty patterns. + // Return % which always matches instead. + "" => "%", + + string s => methodType switch + { + StartsEndsWithContains.StartsWith => EscapeLikePattern(s) + '%', + StartsEndsWithContains.EndsWith => '%' + EscapeLikePattern(s), + StartsEndsWithContains.Contains => $"%{EscapeLikePattern(s)}%", + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + }, + + char s when !IsLikeWildChar(s) => methodType switch + { + StartsEndsWithContains.StartsWith => s + "%", + StartsEndsWithContains.EndsWith => "%" + s, + StartsEndsWithContains.Contains => $"%{s}%", + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + }, + + char s => methodType switch + { + StartsEndsWithContains.StartsWith => EscapeLikePattern(s.ToString()) + "%", + StartsEndsWithContains.EndsWith => "%" + EscapeLikePattern(s.ToString()), + StartsEndsWithContains.Contains => $"%{EscapeLikePattern(s.ToString())}%", + _ => throw new ArgumentOutOfRangeException(nameof(methodType), methodType, null) + }, + + _ => throw new UnreachableException() + }; + + public enum StartsEndsWithContains + { + StartsWith, + EndsWith, + Contains + } + + //Extra resources + // https://support.microsoft.com/en-us/office/like-operator-b2f7ef03-9085-4ffb-9829-eef18358e931 + // https://support.microsoft.com/en-us/office/access-wildcard-character-reference-af00c501-7972-40ee-8889-e18abaad12d1 + // https://support.microsoft.com/en-us/office/use-wildcards-in-queries-and-parameters-in-access-ec057a45-78b1-4d16-8c20-242cde582e0b + //These are the characters to escape in LIKE pattern + private static bool IsLikeWildChar(char c) + => c == '%' || c == '_' || c == '[' || c == '^' || c == '?' || c == '#' || c == '*'; + + private static string EscapeLikePattern(string pattern) + { + var builder = new StringBuilder(); + foreach (var c in pattern) + { + if (IsLikeWildChar(c)) + { + builder.Append('['); + builder.Append(c); + builder.Append(']'); + } + else + { + builder.Append(c); + } + } + + return builder.ToString(); + } + + private Expression TranslateByteArrayElementAccess(Expression array, Expression index, Type resultType) + { + var visitedArray = Visit(array); + var visitedIndex = Visit(index); + + return visitedArray is SqlExpression sqlArray + && visitedIndex is SqlExpression sqlIndex + ? Dependencies.SqlExpressionFactory.Function( + "ASCB", + [ Dependencies.SqlExpressionFactory.Function( + "MIDB", + [ + sqlArray, + Dependencies.SqlExpressionFactory.Add( + Dependencies.SqlExpressionFactory.ApplyDefaultTypeMapping(sqlIndex), + Dependencies.SqlExpressionFactory.Constant(1)), + Dependencies.SqlExpressionFactory.Constant(1) ], + nullable: true, + argumentsPropagateNullability: [true, true, true], + typeof(byte[])) ], + nullable: true, + argumentsPropagateNullability: [true], + typeof(int)) + + : QueryCompilationContext.NotTranslatedExpression; + } + + /// + /// GREATEST(a, b, …), which LibRed's engine evaluates as SQL Server and PostgreSQL do: NULL arguments + /// are ignored, and the result is NULL only when every argument is. + /// + public override SqlExpression? GenerateGreatest(IReadOnlyList expressions, Type resultType) + => GenerateGreatestOrLeast("GREATEST", expressions, resultType); + + /// + /// LEAST(a, b, …), which LibRed's engine evaluates as SQL Server and PostgreSQL do: NULL arguments are + /// ignored, and the result is NULL only when every argument is. + /// + public override SqlExpression? GenerateLeast(IReadOnlyList expressions, Type resultType) + => GenerateGreatestOrLeast("LEAST", expressions, resultType); + + private SqlExpression? GenerateGreatestOrLeast(string name, IReadOnlyList expressions, Type resultType) + { + if (expressions.Count == 0) + { + return null; + } + + IReadOnlyList mappedExpressions = ApplyMinMaxResultTypeMapping(expressions, resultType); + + // A NULL argument is skipped rather than propagated, so no argument's nullability carries into the result + // on its own; the result is still nullable, for when every argument is NULL. + return _sqlExpressionFactory.Function( + name, + mappedExpressions, + nullable: true, + argumentsPropagateNullability: Enumerable.Repeat(false, mappedExpressions.Count), + resultType, + ExpressionExtensions.InferTypeMapping(mappedExpressions.ToArray())); + } + + /// + /// Math.Min/Max operands can arrive with different store mappings even though their CLR expression has one + /// declared result type. Prefer an operand mapping for that result type and apply it to every argument; + /// otherwise the first argument can incorrectly force (for example) an Int16 mapping onto an Int32 constant + /// in an outer nested Min/Max. + /// + private IReadOnlyList ApplyMinMaxResultTypeMapping( + IReadOnlyList expressions, Type resultType) + { + RelationalTypeMapping? resultMapping = expressions + .Select(e => e.TypeMapping) + .FirstOrDefault(m => m?.ClrType == resultType); + + if (resultMapping is null) + { + return expressions; + } + + return expressions + .Select(e => Dependencies.SqlExpressionFactory.ApplyTypeMapping(e, resultMapping)) + .ToArray(); + } + + + private static string? GetProviderType(SqlExpression expression) + => expression.TypeMapping?.StoreType; + +} diff --git a/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitorFactory.cs b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitorFactory.cs new file mode 100644 index 000000000..d76e2bbc7 --- /dev/null +++ b/src/LibRed/LibRed.EFCore/Query/Internal/LibRedSqlTranslatingExpressionVisitorFactory.cs @@ -0,0 +1,35 @@ +using EntityFrameworkCore.Jet.Query.Internal; +using EntityFrameworkCore.LibRed.Infrastructure; +using EntityFrameworkCore.LibRed.Infrastructure.Internal; + +namespace EntityFrameworkCore.LibRed.Query.Internal +{ + /// + /// Creates the SQL translating expression visitor for the configured SQL mode: the shared + /// in mode, whose + /// SQL ACE also has to run, and LibRed's own in + /// mode. + /// + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public class LibRedSqlTranslatingExpressionVisitorFactory( + RelationalSqlTranslatingExpressionVisitorDependencies dependencies, + ILibRedOptions options) : IRelationalSqlTranslatingExpressionVisitorFactory + { + /// + /// Relational provider-specific dependencies for this service. + /// + protected virtual RelationalSqlTranslatingExpressionVisitorDependencies Dependencies { get; } = dependencies; + + public virtual RelationalSqlTranslatingExpressionVisitor Create( + QueryCompilationContext queryCompilationContext, + QueryableMethodTranslatingExpressionVisitor queryableMethodTranslatingExpressionVisitor) + => options.SqlMode == LibRedSqlMode.Compatible + ? new JetSqlTranslatingExpressionVisitor(Dependencies, queryCompilationContext, queryableMethodTranslatingExpressionVisitor) + : new LibRedSqlTranslatingExpressionVisitor(Dependencies, queryCompilationContext, queryableMethodTranslatingExpressionVisitor); + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/AccessTypeMapper.cs b/src/LibRed/LibRed.Engine/Execution/AccessTypeMapper.cs index 5e789aca7..a4b1dcb98 100644 --- a/src/LibRed/LibRed.Engine/Execution/AccessTypeMapper.cs +++ b/src/LibRed/LibRed.Engine/Execution/AccessTypeMapper.cs @@ -15,7 +15,7 @@ internal static class AccessTypeMapper { public static ColumnSpec ToColumnSpec(ColumnDefinition column, JetVersion version) { - ColumnSpec spec = MapType(column, version) with + ColumnSpec spec = WithIdentity(MapType(column, version), column.Identity) with { IsNullable = !column.NotNull, // WITH COMPRESSION is only meaningful on the two types ACE accepts it for; Access rejects it @@ -55,6 +55,19 @@ public static ColumnSpec ToColumnSpec(ColumnDefinition column, JetVersion versio return ColumnSpec.Calculated(spec.Name, spec.Type, expression); } + /// + /// Applies a trailing IDENTITY [(seed [, increment])] the way ACE does: a Long column becomes an + /// AutoNumber counting from the attribute's seed by its increment, each 1 when omitted. Those replace whatever + /// the type declared — COUNTER(5, 2) IDENTITY(9, 3) counts 9, 12, 15 and COUNTER(5, 2) IDENTITY + /// counts 1, 2, 3. Every other type is left exactly as declared: SHORT, BIGINT, TEXT(10), + /// GUID and the rest accept the word and ignore it (all verified). + /// + internal static ColumnSpec WithIdentity(ColumnSpec spec, IdentityAttribute? identity) + { + if (identity is null || spec.Type != JetDataType.Int32) return spec; + return spec with { IsAutoNumber = true, Seed = identity.Seed ?? 1, Increment = identity.Increment ?? 1 }; + } + /// Whether WITH COMPRESSION applies to this column's type. Mapped at the HIGHEST version so the /// version gate inside cannot fire here: asking at Version4 made /// BIGINT … WITH COMPRESSION on an ACE 16 file — which had already passed the real gate — report @@ -99,9 +112,9 @@ private static ColumnSpec MapType(ColumnDefinition column, JetVersion version) return t switch { // AutoNumber. COUNTER(seed, increment) parses seed/increment as the (size, scale) pair; a plain - // COUNTER defaults to 1/1. INTEGER IDENTITY(seed, increment) is the ANSI-style spelling. + // COUNTER defaults to 1/1. IDENTITY after another type — INTEGER IDENTITY(seed, increment) — is a + // column attribute rather than part of the type name, applied by WithIdentity. "COUNTER" or "AUTOINCREMENT" or "IDENTITY" - or "INTEGER IDENTITY" or "INT IDENTITY" or "LONG IDENTITY" => new ColumnSpec(column.Name, JetDataType.Int32, 4, IsFixedLength: true, IsAutoNumber: true, Seed: column.Size ?? 1, Increment: column.Scale ?? 1), "INTEGER" or "INT" or "LONG" or "INTEGER4" diff --git a/src/LibRed/LibRed.Engine/Execution/EvalScope.cs b/src/LibRed/LibRed.Engine/Execution/EvalScope.cs index e86e37a37..239ee7413 100644 --- a/src/LibRed/LibRed.Engine/Execution/EvalScope.cs +++ b/src/LibRed/LibRed.Engine/Execution/EvalScope.cs @@ -64,6 +64,13 @@ public bool TryResolve(ColumnReference reference, out object? value) internal static string Describe(ColumnReference r) => r.Table is null ? r.Column : $"{r.Table}.{r.Column}"; + /// The current row's value at a 1-based column position. + public object? At(int position) => + position >= 1 && position <= row.Length + ? row[position - 1] + : throw new InvalidOperationException( + $"'{position}' is not a valid field name or expression: ORDER BY {position} names no output column."); + /// Every table alias visible in this scope and its enclosing scopes — so index selection on a /// correlated subquery knows which column references belong to the outer query (and are seekable constants). public IEnumerable VisibleAliases() diff --git a/src/LibRed/LibRed.Engine/Execution/ExistsSemiJoin.cs b/src/LibRed/LibRed.Engine/Execution/ExistsSemiJoin.cs index 65e8c6d2c..7c9365fd9 100644 --- a/src/LibRed/LibRed.Engine/Execution/ExistsSemiJoin.cs +++ b/src/LibRed/LibRed.Engine/Execution/ExistsSemiJoin.cs @@ -1,3 +1,4 @@ +using EntityFrameworkCore.Jet.Data; using LibRed.Catalog; using LibRed.Engine.Plan; using LibRed.Engine.Planning; @@ -198,7 +199,7 @@ private static bool TopCannotChangeExistence(SelectStatement subquery) return !subquery.TopPercent && subquery.Top is LiteralExpression { Value: { } value } && IsNumeric(value) - && Convert.ToDecimal(value, System.Globalization.CultureInfo.InvariantCulture) >= 1m; + && JetDecimalConverter.ToDecimal(value, System.Globalization.CultureInfo.InvariantCulture) >= 1m; } private static bool IsNumeric(object v) diff --git a/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.Format.cs b/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.Format.cs new file mode 100644 index 000000000..039363896 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.Format.cs @@ -0,0 +1,840 @@ +using System.Globalization; +using System.Text; +using LibRed.Sql.Ast; + +namespace LibRed.Engine.Execution; + +/// +/// Access Format, FormatNumber, FormatCurrency, FormatPercent and FormatDateTime +/// (verified vs ACE). Output follows the current culture's separators, names and patterns, as ACE follows the +/// system's regional settings. +/// +internal sealed partial class ExpressionEvaluator +{ + private enum FormatKind + { + Number, + Date, + Text, + } + + private enum NumberStyle + { + Number, + Currency, + Percent, + } + + /// + /// Access Format(value, [format], [firstdayofweek], [firstweekofyear]). A named format is matched whole and + /// without regard to case. Otherwise the format is a text format when it has @ & < > !, a number + /// format when it has a 0 or #, a date format when it has a date or time symbol, and otherwise a + /// number format of literals only. Text that is neither a number nor a date is returned unchanged by number and + /// date formats. A Null value is empty text, or the format's Null section. A Null format or setting gives Null + /// where ACE raises an error. + /// + private object? FormatValue(FunctionCall f) + { + object? value = Evaluate(f.Arguments[0]); + string format = ""; + if (f.Arguments.Count > 1) + { + if (Evaluate(f.Arguments[1]) is not { } formatValue) + return null; + format = ConcatText(formatValue); + } + if (FirstDayOfWeek(f, 2) is not { } first || FirstWeekOfYear(f, 3) is not { } rule) + return null; + return FormatText(value, format, first, rule); + } + + /// The date formats of FormatDateTime's 0 to 4 — the general date, long date, short date, long time and + /// short time — which the named formats of the same names share. + private static readonly string[] DateTimeFormats = ["c", "dddddd", "ddddd", "ttttt", "hh:nn"]; + + private static string FormatText(object? value, string format, DayOfWeek first, CalendarWeekRule rule) + { + if (format.Length == 0) + return value is null ? "" : GeneralText(value); + + switch (format.ToLowerInvariant()) + { + case "general number": return NamedNumber(value, number => ConcatText(number)); + case "currency": return NamedNumber(value, number => StyledNumber(DigitsOf(number), NumberStyle.Currency, -1, -2, -2, -2)); + case "fixed": return FormatNumberSections(value, ["0.00"]); + case "standard": return FormatNumberSections(value, ["#,##0.00"]); + case "percent": return FormatNumberSections(value, ["0.00%"]); + case "scientific": return FormatNumberSections(value, ["0.00E+00"]); + case "yes/no": return NamedNumber(value, number => DigitsOf(number).IsZero ? "No" : "Yes"); + case "true/false": return NamedNumber(value, number => DigitsOf(number).IsZero ? "False" : "True"); + case "on/off": return NamedNumber(value, number => DigitsOf(number).IsZero ? "Off" : "On"); + case "general date": return FormatDate(value, DateTimeFormats[0], first, rule); + case "long date": return FormatDate(value, DateTimeFormats[1], first, rule); + case "medium date": return FormatDate(value, "dd-mmm-yy", first, rule); + case "short date": return FormatDate(value, DateTimeFormats[2], first, rule); + case "long time": return FormatDate(value, DateTimeFormats[3], first, rule); + case "medium time": return FormatDate(value, "hh:nn AM/PM", first, rule); + case "short time": return FormatDate(value, DateTimeFormats[4], first, rule); + } + + List sections = FormatSections(format); + return FormatKindOf(format) switch + { + FormatKind.Text => FormatTextSections(value, sections), + FormatKind.Date => FormatDate(value, sections[0], first, rule), + _ => FormatNumberSections(value, sections), + }; + } + + /// A value as Format writes it without a format: as CStr does, but a date with its year padded and + /// rounded to the second. + private static string GeneralText(object value) => + value is DateTime date ? DateText(RoundToSecond(date), padYear: true) : ConcatText(value); + + private static string NamedNumber(object? value, Func write) => + value is null or string { Length: 0 } ? "" + : FormatNumberOperand(value) is { } number ? write(number) + : ConcatText(value); + + /// The number a value stands for in a number format: a number, a Boolean, a date's serial, or text that + /// reads as a number or else as a date. Null for other text, which the format leaves as it is. + private static object? FormatNumberOperand(object value) => value switch + { + string or char => TryTextAsNumber(value.ToString()!) is { } number ? number + : VbaDateText.TryParse(value.ToString()!, CultureInfo.CurrentCulture, out DateTime date) ? date.ToOADate() + : null, + DateTime date => date.ToOADate(), + bool b => b ? -1 : 0, + Guid or byte[] => null, + _ => value, + }; + + /// The date a value stands for in a date format, or null for text that is neither a number nor a date. + /// A number past the range of a date is an overflow. + private static DateTime? FormatDateOperand(object value) => + value is DateTime date ? date + : FormatNumberOperand(value) is { } number ? OaDate(Dbl(number)) + : null; + + /// A date rounded to the nearest second, as ACE writes it. + private static DateTime RoundToSecond(DateTime date) + { + long rest = date.Ticks % TimeSpan.TicksPerSecond; + long ticks = date.Ticks - rest + (rest >= TimeSpan.TicksPerSecond / 2 ? TimeSpan.TicksPerSecond : 0); + return ticks <= DateTime.MaxValue.Ticks ? new DateTime(ticks, date.Kind) : date; + } + + /// The format's sections, split at semicolons outside quotes and escapes. + private static List FormatSections(string format) + { + var sections = new List(); + int start = 0; + for (int i = 0; i < format.Length; i++) + { + if (format[i] == '"') + i = ClosingQuote(format, i); + else if (format[i] == '\\') + i++; + else if (format[i] == ';') + { + sections.Add(format[start..i]); + start = i + 1; + } + } + sections.Add(format[start..]); + return sections; + } + + private static int ClosingQuote(string format, int open) + { + int close = format.IndexOf('"', open + 1); + return close < 0 ? format.Length : close; + } + + private static FormatKind FormatKindOf(string format) + { + bool digits = false, date = false; + for (int i = 0; i < format.Length; i++) + { + char c = format[i]; + if (c == '"') + i = ClosingQuote(format, i); + else if (c == '\\') + i++; + else if (c is '@' or '&' or '<' or '>' or '!') + return FormatKind.Text; + else if (c is '0' or '#') + digits = true; + else if (c is '/' or ':' || DateSymbolLength(format, i) > 0 || Meridiem(format, i) > 0) + date = true; + } + return digits || !date ? FormatKind.Number : FormatKind.Date; + } + + // ---- Text formats ---- + + /// + /// A text format: @ is a character or a space and & a character or nothing, filled from the right + /// unless ! fills them from the left; < and > force lower and upper case (both together + /// change nothing). Characters the placeholders do not take follow the format's output (left to right, the + /// leftmost are dropped instead). Empty text and Null use the second section, or are empty. + /// + private static string FormatTextSections(object? value, List sections) + { + string text = value is null ? "" : GeneralText(value); + if (text.Length == 0) + return sections.Count > 1 ? FormatTextSection(sections[1], "") : ""; + return FormatTextSection(sections[0], text); + } + + private static string FormatTextSection(string section, string text) + { + var items = new List<(char Placeholder, string Literal)>(); + bool lower = false, upper = false, leftToRight = false; + for (int i = 0; i < section.Length; i++) + { + char c = section[i]; + switch (c) + { + case '"': + int close = ClosingQuote(section, i); + items.Add(('\0', section[(i + 1)..close])); + i = close; + break; + case '\\': + if (i + 1 < section.Length) + items.Add(('\0', section[++i].ToString())); + break; + case '@' or '&': + items.Add((c, "")); + break; + case '<': lower = true; break; + case '>': upper = true; break; + case '!': leftToRight = true; break; + default: + items.Add(('\0', c.ToString())); + break; + } + } + if (lower != upper) + text = lower ? text.ToLowerInvariant() : text.ToUpperInvariant(); + + int placeholders = items.Count(item => item.Placeholder != '\0'); + var output = new StringBuilder(); + if (placeholders == 0) + { + foreach (var item in items) + output.Append(item.Literal); + return output.Append(text).ToString(); + } + + if (leftToRight && text.Length > placeholders) + text = text[^placeholders..]; + int blanks = leftToRight ? 0 : Math.Max(0, placeholders - text.Length); + int slot = 0, next = 0; + foreach (var item in items) + { + if (item.Placeholder == '\0') + { + output.Append(item.Literal); + continue; + } + if (slot++ < blanks || next >= text.Length) + { + if (item.Placeholder == '@') + output.Append(' '); + } + else + output.Append(text[next++]); + } + return output.Append(text[next..]).ToString(); + } + + // ---- Date formats ---- + + private static string FormatDate(object? value, string format, DayOfWeek first, CalendarWeekRule rule) + { + if (value is null or string { Length: 0 }) + return ""; + return FormatDateOperand(value) is { } date + ? DateSymbols(RoundToSecond(date), format, first, rule) + : ConcatText(value); + } + + /// + /// A date written by a date format. Symbols are matched without regard to case, longest first, and a run longer + /// than a symbol continues as the next (yyy is yy then y). m and mm are the minute + /// straight after an hour symbol. AM/PM, A/P and AMPM make the hour a 12-hour one. Anything else is written as it is. + /// + private static string DateSymbols(DateTime date, string format, DayOfWeek first, CalendarWeekRule rule) + { + CultureInfo culture = CultureInfo.CurrentCulture; + DateTimeFormatInfo names = culture.DateTimeFormat; + bool twelveHour = false; + for (int i = 0; i < format.Length; i++) + twelveHour |= Meridiem(format, i) > 0; + + var output = new StringBuilder(); + bool afterHour = false; + for (int i = 0; i < format.Length;) + { + char c = format[i]; + if (c == '"') + { + int close = ClosingQuote(format, i); + output.Append(format, i + 1, close - i - 1); + i = close + 1; + continue; + } + if (c == '\\') + { + if (i + 1 < format.Length) + output.Append(format[i + 1]); + i += 2; + continue; + } + if (Meridiem(format, i) is var meridiem and > 0) + { + // AM/PM and A/P take the case of their first letter; AMPM is the regional designator as it is defined. + bool morning = date.Hour < 12; + string text = meridiem switch + { + 4 => morning ? names.AMDesignator : names.PMDesignator, + 5 => morning ? "AM" : "PM", + _ => morning ? "A" : "P", + }; + output.Append(meridiem != 4 && char.IsLower(c) ? text.ToLowerInvariant() : text); + i += meridiem; + afterHour = false; + continue; + } + int length = DateSymbolLength(format, i); + if (length == 0) + { + output.Append(c switch + { + '/' => names.DateSeparator, + ':' => names.TimeSeparator, + _ => c.ToString(), + }); + i++; + continue; + } + + char symbol = char.ToLowerInvariant(c); + int hour12 = date.Hour % 12 == 0 ? 12 : date.Hour % 12; + output.Append((symbol, length) switch + { + ('d', 1) => date.Day.ToString(culture), + ('d', 2) => date.Day.ToString("00", culture), + ('d', 3) => names.GetAbbreviatedDayName(date.DayOfWeek), + ('d', 4) => names.GetDayName(date.DayOfWeek), + ('d', 5) => date.ToString(names.ShortDatePattern, culture), + ('d', _) => date.ToString(names.LongDatePattern, culture), + ('m', 1) when afterHour => date.Minute.ToString(culture), + ('m', 2) when afterHour => date.Minute.ToString("00", culture), + ('m', 1) => date.Month.ToString(culture), + ('m', 2) => date.Month.ToString("00", culture), + ('m', 3) => names.GetAbbreviatedMonthName(date.Month), + ('m', _) => names.GetMonthName(date.Month), + ('y', 1) => date.DayOfYear.ToString(culture), + ('y', 2) => (date.Year % 100).ToString("00", culture), + ('y', _) => date.Year.ToString("0000", culture), + ('h', 1) => (twelveHour ? hour12 : date.Hour).ToString(culture), + ('h', _) => (twelveHour ? hour12 : date.Hour).ToString("00", culture), + ('n', 1) => date.Minute.ToString(culture), + ('n', _) => date.Minute.ToString("00", culture), + ('s', 1) => date.Second.ToString(culture), + ('s', _) => date.Second.ToString("00", culture), + ('w', 1) => (DaysIntoWeek(date, first) + 1).ToString(culture), + ('w', _) => WeekOfYear(date, first, rule).ToString(culture), + ('q', _) => ((date.Month + 2) / 3).ToString(culture), + ('c', _) => DateText(date, padYear: true), + _ => date.ToString(LongTimePattern(names), culture), + }); + afterHour = symbol == 'h'; + i += length; + } + return output.ToString(); + } + + /// The length of the date symbol at , or 0 when there is none. + private static int DateSymbolLength(string format, int index) + { + char symbol = char.ToLowerInvariant(format[index]); + int run = 1; + while (index + run < format.Length && char.ToLowerInvariant(format[index + run]) == symbol) + run++; + return symbol switch + { + 'd' => Math.Min(run, 6), + 'm' => Math.Min(run, 4), + 'y' => run >= 4 ? 4 : run >= 2 ? 2 : 1, + 'h' or 'n' or 's' or 'w' => Math.Min(run, 2), + 'q' or 'c' => 1, + 't' => run >= 5 ? 5 : 0, + _ => 0, + }; + } + + /// The length of the AM/PM (5), AMPM (4) or A/P (3) symbol at , or 0. + private static int Meridiem(string format, int index) + { + ReadOnlySpan rest = format.AsSpan(index); + return rest.StartsWith("am/pm", StringComparison.OrdinalIgnoreCase) ? 5 + : rest.StartsWith("ampm", StringComparison.OrdinalIgnoreCase) ? 4 + : rest.StartsWith("a/p", StringComparison.OrdinalIgnoreCase) ? 3 + : 0; + } + + // ---- Number formats ---- + + /// + /// A number format of up to four sections: positive, negative, zero and Null. A negative value with no negative + /// section uses the first with a minus sign. A value that the section rounds to zero is a zero, and a zero uses the + /// third section, or the first when there is none or it is empty. An empty first section writes nothing. + /// + private static string FormatNumberSections(object? value, List sections) + { + string Section(int i) => i < sections.Count ? sections[i] : ""; + if (value is null) + return sections.Count > 3 ? new NumberSection(sections[3]).Write(default) : ""; + if (value is string { Length: 0 }) + return ""; + if (FormatNumberOperand(value) is not { } number) + return ConcatText(value); + + FormatDigits digits = DigitsOf(number); + if (!digits.IsZero) + { + bool negativeSection = digits.Negative && Section(1).Length > 0; + var section = new NumberSection(negativeSection ? Section(1) : Section(0)); + if (section.Text.Length == 0) + return ""; + FormatDigits magnitude = digits with { Negative = false }; + if (!section.RoundsToZero(magnitude)) + return (digits.Negative && !negativeSection ? "-" : "") + section.Write(magnitude); + } + string zero = Section(2).Length > 0 ? Section(2) : Section(0); + return new NumberSection(zero).Write(default); + } + + /// + /// A number as its decimal digits: 0.Digits × 10^Point, with no leading or trailing zeros (none at all for + /// zero). A Double has 15 significant digits and a Single 7, as ACE writes them; other numbers are exact. + /// + private readonly record struct FormatDigits(bool Negative, string Digits, int Point) + { + private readonly string? digits = Digits; + + public string Digits + { + get => digits ?? ""; + init => digits = value; + } + + public bool IsZero => Digits.Length == 0; + + public FormatDigits Scale(int powerOfTen) => IsZero ? this : this with { Point = Point + powerOfTen }; + + /// Rounded to places, half away from zero. + public FormatDigits Round(int decimals) + { + int keep = Point + decimals; + if (keep >= Digits.Length) + return this; + if (keep < 0) + return this with { Digits = "", Point = 0 }; + char[] kept = Digits[..keep].ToCharArray(); + int point = Point; + if (Digits[keep] >= '5') + { + int i = kept.Length - 1; + while (i >= 0 && kept[i] == '9') + kept[i--] = '0'; + if (i >= 0) + kept[i]++; + else + { + kept = ['1', .. kept]; + point++; + } + } + string rounded = new string(kept).TrimEnd('0'); + return this with { Digits = rounded, Point = rounded.Length == 0 ? 0 : point }; + } + + /// The digits before the point ("" below 1). + public string Whole => Point <= 0 ? "" : Digits.Length >= Point ? Digits[..Point] : Digits.PadRight(Point, '0'); + + /// The first digits after the point. + public string Fraction(int count) + { + var fraction = new char[count]; + for (int i = 0; i < count; i++) + { + int index = Point + i; + fraction[i] = index >= 0 && index < Digits.Length ? Digits[index] : '0'; + } + return new string(fraction); + } + } + + private static FormatDigits DigitsOf(object number) => Numeric(number) switch + { + double d => ScientificDigits(d.ToString("E14", CultureInfo.InvariantCulture)), + float f => ScientificDigits(((double)f).ToString("E6", CultureInfo.InvariantCulture)), + var n => PlainDigits(Convert.ToString(n, CultureInfo.InvariantCulture)!), + }; + + private static FormatDigits ScientificDigits(string text) + { + bool negative = text[0] == '-'; + int mark = text.IndexOf('E'); + string mantissa = text[(negative ? 1 : 0)..mark].Replace(".", "").TrimEnd('0'); + int exponent = int.Parse(text[(mark + 1)..], CultureInfo.InvariantCulture); + return mantissa.Length == 0 ? default : new(negative, mantissa, exponent + 1); + } + + private static FormatDigits PlainDigits(string text) + { + bool negative = text[0] == '-'; + string unsigned = negative ? text[1..] : text; + int dot = unsigned.IndexOf('.'); + string digits = unsigned.Replace(".", ""); + int point = dot < 0 ? unsigned.Length : dot; + int leading = digits.Length - digits.TrimStart('0').Length; + digits = digits.TrimStart('0').TrimEnd('0'); + return digits.Length == 0 ? default : new(negative, digits, point - leading); + } + + /// + /// One section of a number format. 0 is a digit or a zero and # a digit or nothing; the leftmost + /// integer placeholder takes any further digits. The first . is the decimal point. A comma after an integer + /// placeholder groups thousands when another integer placeholder follows it, and otherwise divides by 1000; a + /// comma before any placeholder is written, one after the point is dropped. Each % multiplies by 100. + /// E+ E- e+ e- after a placeholder give scientific notation with as many integer digits as there are integer + /// placeholders; an E without a sign is dropped. […] is dropped, and * drops itself and the + /// next character. : and / are the regional separators; anything else is written as it is. + /// + private sealed class NumberSection + { + private enum Kind + { + Literal, + Zero, + Hash, + Point, + FractionZero, + FractionHash, + Exponent, + ExponentDigit, + } + + private readonly List<(Kind Kind, string Text)> items = []; + private readonly int wholePlaceholders; + private readonly int fractionPlaceholders; + private readonly int minimumWhole; + private readonly int scale; + private readonly bool group; + private readonly bool scientific; + private readonly int exponentZeros; + + public string Text { get; } + + public NumberSection(string text) + { + Text = text; + NumberFormatInfo format = CultureInfo.CurrentCulture.NumberFormat; + DateTimeFormatInfo dates = CultureInfo.CurrentCulture.DateTimeFormat; + bool fraction = false, exponent = false; + int percent = 0, trailingCommas = 0, firstZero = -1; + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + switch (c) + { + case '"': + int close = ClosingQuote(text, i); + Literal(text[(i + 1)..close]); + i = close; + break; + case '\\': + if (i + 1 < text.Length) + Literal(text[++i].ToString()); + break; + case '[': + int end = text.IndexOf(']', i); + i = end < 0 ? text.Length : end; + break; + case '*': + i++; + break; + case '0' or '#': + if (exponent) + { + items.Add((Kind.ExponentDigit, "")); + if (c == '0') exponentZeros++; + } + else if (fraction) + { + items.Add((c == '0' ? Kind.FractionZero : Kind.FractionHash, "")); + fractionPlaceholders++; + } + else + { + items.Add((c == '0' ? Kind.Zero : Kind.Hash, "")); + if (c == '0' && firstZero < 0) firstZero = wholePlaceholders; + wholePlaceholders++; + if (trailingCommas > 0) group = true; + trailingCommas = 0; + } + break; + case '.' when !fraction && !exponent: + items.Add((Kind.Point, format.NumberDecimalSeparator)); + fraction = true; + break; + case ',': + if (!fraction && !exponent && wholePlaceholders == 0) + Literal(","); + else if (!fraction && !exponent) + trailingCommas++; + break; + case '%': + percent++; + Literal("%"); + break; + case 'E' or 'e': + if (i + 1 < text.Length && text[i + 1] is '+' or '-' && !exponent && wholePlaceholders + fractionPlaceholders > 0) + { + items.Add((Kind.Exponent, $"{c}{text[i + 1]}")); + exponent = true; + scientific = true; + i++; + } + else if (i + 1 < text.Length && text[i + 1] is '+' or '-') + Literal(c.ToString()); + break; + case ':': + Literal(dates.TimeSeparator); + break; + case '/': + Literal(dates.DateSeparator); + break; + default: + Literal(c.ToString()); + break; + } + } + scale = 2 * percent - 3 * trailingCommas; + // A 0 placeholder shows a digit even for a zero, and so does every placeholder to its right. + minimumWhole = firstZero < 0 ? 0 : wholePlaceholders - firstZero; + } + + private void Literal(string text) => items.Add((Kind.Literal, text)); + + private bool HasDigits => wholePlaceholders + fractionPlaceholders > 0; + + /// Whether the section writes the value as zero. + public bool RoundsToZero(FormatDigits value) => + HasDigits && !scientific && value.Scale(scale).Round(fractionPlaceholders).IsZero; + + public string Write(FormatDigits value) + { + NumberFormatInfo format = CultureInfo.CurrentCulture.NumberFormat; + value = value.Scale(scale); + string whole, fractionDigits, exponentText = ""; + if (scientific) + { + int wholeCount = Math.Max(wholePlaceholders, 1); + int exponent = 0; + FormatDigits mantissa = value; + if (!value.IsZero) + { + exponent = value.Point - wholeCount; + mantissa = value.Scale(-exponent).Round(fractionPlaceholders); + if (mantissa.Point > wholeCount) + { + exponent++; + mantissa = value.Scale(-exponent).Round(fractionPlaceholders); + } + } + whole = mantissa.Whole.PadLeft(wholePlaceholders, '0'); + fractionDigits = mantissa.Fraction(fractionPlaceholders); + exponentText = Math.Abs(exponent).ToString(CultureInfo.InvariantCulture).PadLeft(exponentZeros, '0'); + if (exponent < 0) exponentText = "-" + exponentText; + else if (items.Find(item => item.Kind == Kind.Exponent).Text[1] == '+') exponentText = "+" + exponentText; + } + else + { + value = value.Round(fractionPlaceholders); + whole = value.Whole.PadLeft(minimumWhole, '0'); + fractionDigits = value.Fraction(fractionPlaceholders); + } + + int keepFraction = fractionPlaceholders; + var fractionKinds = items.Where(item => item.Kind is Kind.FractionZero or Kind.FractionHash).Select(item => item.Kind).ToList(); + while (keepFraction > 0 && fractionKinds[keepFraction - 1] == Kind.FractionHash && fractionDigits[keepFraction - 1] == '0') + keepFraction--; + + var output = new StringBuilder(); + int wholeSlot = 0, fractionSlot = 0; + bool exponentWritten = false; + foreach (var (kind, text) in items) + { + switch (kind) + { + case Kind.Literal: + output.Append(text); + break; + case Kind.Zero or Kind.Hash: + // Digits are counted from the right; the leftmost placeholder also takes every digit past it. + int place = wholePlaceholders - 1 - wholeSlot++; + int highest = place == wholePlaceholders - 1 ? whole.Length - 1 : Math.Min(place, whole.Length - 1); + for (int digit = highest; digit >= place; digit--) + { + output.Append(whole[whole.Length - 1 - digit]); + if (group && digit > 0 && digit % 3 == 0) + output.Append(format.NumberGroupSeparator); + } + break; + case Kind.Point: + if (wholePlaceholders == 0) + output.Append(whole); + output.Append(text); + break; + case Kind.FractionZero or Kind.FractionHash: + if (wholePlaceholders == 0 && fractionSlot == 0 && !items.Exists(item => item.Kind == Kind.Point)) + output.Append(whole); + if (fractionSlot < keepFraction) + output.Append(fractionDigits[fractionSlot]); + fractionSlot++; + break; + case Kind.Exponent: + output.Append(text[0]); + break; + case Kind.ExponentDigit: + if (!exponentWritten) + output.Append(exponentText); + exponentWritten = true; + break; + } + } + if (scientific && !exponentWritten) + output.Append(exponentText); + return output.ToString(); + } + } + + // ---- FormatNumber, FormatCurrency, FormatPercent ---- + + /// + /// Access FormatNumber, FormatCurrency and FormatPercent(value, [digits], [leading zero], + /// [parentheses], [group digits]) (verified vs ACE). The value is read as the conversion functions read it and + /// rounded half away from zero; a value that rounds to zero has no sign. Digits of -1 take the regional default, + /// and below that are an invalid procedure call. The other settings are -1 (yes), 0 (no) or -2 (the regional + /// setting), and anything else is an invalid procedure call. A Null value is empty text; a Null setting gives Null + /// where ACE raises a type mismatch. + /// + private object? FormatStyled(FunctionCall f, NumberStyle style) + { + int[] settings = [-1, -2, -2, -2]; + for (int i = 1; i < f.Arguments.Count; i++) + { + if (Evaluate(f.Arguments[i]) is not { } setting) + return null; + settings[i - 1] = i == 1 ? Setting(setting, -1, short.MaxValue) : Setting(setting, -2, 0); + } + if (Evaluate(f.Arguments[0]) is not { } value) + return ""; + return StyledNumber(DigitsOf(ConversionNumber(value)), style, settings[0], settings[1], settings[2], settings[3]); + } + + // The .NET pattern numbers of NumberFormatInfo: n is the number, $ and % the symbol, - the negative sign. + private static readonly string[] CurrencyPositive = ["$n", "n$", "$ n", "n $"]; + private static readonly string[] CurrencyNegative = + ["($n)", "-$n", "$-n", "$n-", "(n$)", "-n$", "n-$", "n$-", "-n $", "-$ n", "n $-", "$ n-", "$ -n", "n- $", "($ n)", "(n $)", "$- n"]; + private static readonly string[] PercentPositive = ["n %", "n%", "%n", "% n"]; + private static readonly string[] PercentNegative = ["-n %", "-n%", "-%n", "%-n", "%n-", "n-%", "n%-", "-% n", "n %-", "% n-", "% -n", "n- %"]; + private static readonly string[] NumberNegative = ["(n)", "-n", "- n", "n-", "n -"]; + + private static string StyledNumber(FormatDigits value, NumberStyle style, int digits, int leadingZero, int parentheses, int groupDigits) + { + NumberFormatInfo format = CultureInfo.CurrentCulture.NumberFormat; + // The default digits are the regional setting, which is 2 for numbers; .NET's ICU data says 3 for many cultures + // (en-US included), so only the currency's own digits are taken from the culture. + if (digits < 0) + digits = style == NumberStyle.Currency ? format.CurrencyDecimalDigits : 2; + if (style == NumberStyle.Percent) + value = value.Scale(2); + value = value.Round(digits); + + (string point, string separator) = style switch + { + NumberStyle.Currency => (format.CurrencyDecimalSeparator, format.CurrencyGroupSeparator), + NumberStyle.Percent => (format.PercentDecimalSeparator, format.PercentGroupSeparator), + _ => (format.NumberDecimalSeparator, format.NumberGroupSeparator), + }; + string whole = value.Whole; + if (whole.Length == 0 && leadingZero != 0) + whole = "0"; + if (groupDigits != 0) + { + var grouped = new StringBuilder(); + for (int i = 0; i < whole.Length; i++) + { + if (i > 0 && (whole.Length - i) % 3 == 0) + grouped.Append(separator); + grouped.Append(whole[i]); + } + whole = grouped.ToString(); + } + string number = digits > 0 ? whole + point + value.Fraction(digits) : whole; + + (string positive, string negative) = style switch + { + NumberStyle.Currency => (CurrencyPositive[format.CurrencyPositivePattern], CurrencyNegative[format.CurrencyNegativePattern]), + NumberStyle.Percent => (PercentPositive[format.PercentPositivePattern], PercentNegative[format.PercentNegativePattern]), + _ => ("n", NumberNegative[format.NumberNegativePattern]), + }; + string pattern = !value.Negative || value.IsZero ? positive + : parentheses == -1 ? "(" + positive + ")" + : parentheses == 0 && negative.Contains('(') ? "-" + positive + : negative; + + string symbol = style switch + { + NumberStyle.Currency => format.CurrencySymbol, + NumberStyle.Percent => format.PercentSymbol, + _ => "", + }; + var output = new StringBuilder(); + foreach (char c in pattern) + { + output.Append(c switch + { + 'n' => number, + '$' or '%' => symbol, + '-' => format.NegativeSign, + _ => c.ToString(), + }); + } + return output.ToString(); + } + + // ---- FormatDateTime ---- + + /// + /// Access FormatDateTime(date, [format]) (verified vs ACE): 0 the general date, 1 the long date, 2 the short + /// date, 3 the long time and 4 the 24-hour hh:mm time. A format outside 0-4 is an invalid procedure call even for + /// a Null date, which is empty text. The date is read as CDate reads it. + /// + private object? FormatDateTime(FunctionCall f) + { + if (Optional(f, 1, 0, v => Setting(v, 0, 4)) is not { } format) + return null; + if (Evaluate(f.Arguments[0]) is not { } value) + return ""; + return DateSymbols(RoundToSecond(ToDate(value)), DateTimeFormats[format], DayOfWeek.Sunday, CalendarWeekRule.FirstDay); + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.cs b/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.cs index ff86a8a0e..7091c49cc 100644 --- a/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.cs +++ b/src/LibRed/LibRed.Engine/Execution/ExpressionEvaluator.cs @@ -1,17 +1,35 @@ using System.Globalization; using System.Text; using System.Text.RegularExpressions; +using EntityFrameworkCore.Jet.Data; using LibRed.Sql.Ast; +using LibRed.Storage; namespace LibRed.Engine.Execution; +/// What ACE takes a result to be, as far as its decimal places go (see +/// ). +internal enum NumberClass +{ + Other, + Whole, + Text, + Date, + Decimal, + Currency, + Double, +} + +/// A result's , with its places when it is a Decimal. +internal readonly record struct NumberType(NumberClass Class, int Places = 0); + /// /// Evaluates an AST against a single row, resolving column /// references through an (which chains to outer scopes for /// correlation). Comparisons coerce numeric operands; SQL nulls propagate (a comparison /// involving null yields null, treated as "not true" by filters). /// -internal sealed class ExpressionEvaluator( +internal sealed partial class ExpressionEvaluator( EvalScope scope, IScalarSubqueryRunner subqueries, ParameterBag? parameters = null, @@ -35,6 +53,7 @@ public ExpressionEvaluator Rebind(object?[] row) ExistsExpression e => subqueries.ExecuteExists(e.Query, scope), InSubqueryExpression i => EvaluateInSubquery(i), InListExpression i => EvaluateInList(i), + BetweenExpression be => EvaluateBetween(be), CaseExpression c => EvaluateCase(c), FunctionCall f => EvaluateFunction(f), UnaryExpression u => EvaluateUnary(u), @@ -43,6 +62,7 @@ public ExpressionEvaluator Rebind(object?[] row) ? parameters.Resolve(p.Name) : throw new InvalidOperationException($"No parameters were supplied for '{p.Name}'."), SystemVariableExpression v => ResolveSystemVariable(v.Name), + OutputColumnPosition p => scope.At(p.Position), _ => throw new NotSupportedException($"Cannot evaluate {expression.GetType().Name}."), }; @@ -117,23 +137,37 @@ private static bool TryNiladicFunction(ColumnReference c, out object? value) return inq.Negated ? (result is null ? null : !result) : result; } - /// x [NOT] IN (a, b, …) over a literal list, evaluated iteratively (not as a recursive OR-tree) - /// so a huge list can't overflow the stack. Same three-valued semantics as the subquery form: NULL if x is null - /// or (no match and some item is null), otherwise the membership result (negated for NOT IN). + /// x [NOT] IN (a, b, …) over a list, evaluated iteratively (not as a recursive OR-tree) so a huge + /// list can't overflow the stack. Null when x is; otherwise whether an item equals x, with each pair brought to a + /// common kind as = does (verified vs ACE). A miss is Null when an item is Null, as the standard has it: + /// 5 IN (1, NULL) and 5 NOT IN (1, NULL) are both Null. This departs from ACE, which skips a Null + /// item and makes them False and True. A True or False item is compared as -1 or 0, not as a truth + /// test. private object? EvaluateInList(InListExpression inl) { object? val = Evaluate(inl.Value); if (val is null) return null; - bool hasNull = false, found = false; + bool found = false, hasNull = false; foreach (Expression itemExpr in inl.Items) { - object? item = Evaluate(itemExpr); - if (item is null) hasNull = true; - else if (Compare(val, item) == 0) { found = true; break; } + if (Evaluate(itemExpr) is not { } item) hasNull = true; + else if (CompareAsKinds(val, item) == 0) { found = true; break; } } - bool? result = found ? true : hasNull ? null : false; - return inl.Negated ? (result is null ? null : !result) : result; + return !found && hasNull ? null : found != inl.Negated; + } + + /// x [NOT] BETWEEN a AND b: whether x lies between the two bounds inclusive, in whichever order + /// they are written, with each pair brought to a common kind as a comparison does. Null when any of the three is + /// (verified vs ACE: 15 BETWEEN NULL AND 10 is Null, not False). + private object? EvaluateBetween(BetweenExpression be) + { + object? val = Evaluate(be.Value), low = Evaluate(be.Low), high = Evaluate(be.High); + if (val is null || low is null || high is null) return null; + + int toLow = CompareAsKinds(val, low), toHigh = CompareAsKinds(val, high); + bool inside = (toLow >= 0 && toHigh <= 0) || (toLow <= 0 && toHigh >= 0); + return inside != be.Negated; } /// Standard SQL CASE. Arms are tested in order and the first whose condition is true wins; @@ -152,6 +186,10 @@ private static bool TryNiladicFunction(ColumnReference c, out object? value) return c.ElseResult is null ? null : Evaluate(c.ElseResult); } + // What Trim, LTrim and RTrim strip (verified vs ACE): the space and the ideographic space U+3000, in any mixture — + // not a tab, CR, LF, no-break space, the other Unicode spaces or a zero-width one. + private static readonly char[] TrimmedSpaces = [' ', '　']; + private object? EvaluateFunction(FunctionCall f) { // Aggregate calls are precomputed per group and resolved by reference — including an outer @@ -163,107 +201,139 @@ private static bool TryNiladicFunction(ColumnReference c, out object? value) // value in the Jet expression service — so a trailing "$" is stripped and dispatched to the base name. string name = f.Name.ToUpperInvariant(); if (name.Length > 1 && name[^1] == '$') name = name[..^1]; + if (f.Filter is not null && !Planning.QueryPlanner.IsAggregate(name)) + throw new InvalidOperationException($"{f.Name} takes no FILTER; only an aggregate does."); ValidateArity(name, f.Arguments.Count); return name switch { - "IIF" => IsTrue(f.Arguments[0]) ? Evaluate(f.Arguments[1]) + "IIF" => Evaluate(f.Arguments[0]) is { } condition && IifCondition(condition) ? Evaluate(f.Arguments[1]) : f.Arguments.Count == 3 ? Evaluate(f.Arguments[2]) : null, "CHOOSE" => Choose(f), "SWITCH" => Switch(f), "NULLIF" => NullIf(f), "COALESCE" => Coalesce(f), - "DATEPART" => DatePart(Evaluate(f.Arguments[0]), Evaluate(f.Arguments[1])), + "GREATEST" => Extreme(f, greatest: true), + "LEAST" => Extreme(f, greatest: false), + "DATEPART" => DatePart(f), "ROUND" => Round(f), - "FIX" => Numeric1(f, Math.Truncate, Math.Truncate), // toward zero - "INT" => Numeric1(f, Math.Floor, Math.Floor), // toward -infinity - "ABS" => Numeric1(f, Math.Abs, Math.Abs), - // VBA/Access type-conversion functions. All propagate NULL. CInt/CLng/CByte round half-to-even - // ("banker's rounding"), which is exactly what Convert.ToInt16/Int32/Byte do. CVar is a no-op - // passthrough (LibRed has no distinct Variant type). - // A Boolean argument goes through Numeric() first, so True converts as VARIANT_BOOL -1 rather than - // .NET's 1 (verified vs ACE: CInt/CLng/CDbl/CSng/CCur(True) are all -1, and CByte(True) overflows - // because a byte cannot hold -1 — which Convert.ToByte(-1) raises for us). - "CCUR" => Convert1(f, v => Math.Round(Dec(v), 4)), // to Currency (decimal, 4 dp) + "FIX" => Numeric1(f, Math.Truncate, Math.Truncate, keepsDate: true), // toward zero + "INT" => Numeric1(f, Math.Floor, Math.Floor, keepsDate: true), // toward -infinity + "ABS" => Numeric1(f, Math.Abs, Math.Abs, keepsDate: false), + // VBA/Access type-conversion functions (verified vs ACE). Each reads its argument as ConversionNumber + // does: text as a number, a date as its serial, True as -1 — so CByte(True) overflows. CInt/CLng/CByte + // round half to even, as Convert.ToInt16/Int32/Byte do, and a value past the type is an overflow. ACE + // raises "Invalid use of Null" for a Null argument; LibRed returns Null. CVar passes its argument + // through (LibRed has no Variant type; ACE hands the value back as text). + "CCUR" => DecimalArgument(f, ToCurrency), "CBOOL" => Convert1(f, v => VbaBool(v)), - "CBYTE" => Convert1(f, v => Convert.ToByte(Numeric(v), CultureInfo.InvariantCulture)), - "CINT" => Convert1(f, v => Convert.ToInt16(Numeric(v), CultureInfo.InvariantCulture)), - "CLNG" => Convert1(f, v => Int(v)), - "CSNG" => Convert1(f, v => Sng(v)), - "CDBL" => Convert1(f, v => Dbl(v)), + "CBYTE" => Convert1(f, v => Convert.ToByte(ConversionNumber(v), CultureInfo.InvariantCulture)), + "CINT" => Convert1(f, v => (short)AsInteger(v)), + "CLNG" => Convert1(f, v => AsLong(v)), + // CLngLng is VBA's LongLong conversion, which the Jet Expression Service does not have (verified: ACE + // reports it undefined), so a LibRed extension. It reads its argument as CLng does, into an Int64. + "CLNGLNG" => Convert1(f, v => AsLongLong(v)), + "CSNG" => Convert1(f, v => Finite(Sng(ConversionNumber(v)))), + "CDBL" => Convert1(f, v => Dbl(ConversionNumber(v))), // CDec has no ACE equivalent — the Jet Expression Service has no such function — so this is a // LibRed extension with no parity contract to honour. CCur is ACE's route to a decimal. - "CDEC" => Convert1(f, v => Dec(v)), - "CSTR" => Convert1(f, VbaString), - "CDATE" => Convert1(f, ToDate), - "CVAR" => Evaluate(f.Arguments[0]), // passthrough (no Variant type) - - // VBA/Access string functions. All propagate NULL; positions are 1-based. Comparisons default to - // case-insensitive (Access "Option Compare Database" = Text), overridable by a compare argument. - // ToText, not ToString: a binary column's value is a UTF-16 STRING to every text function, so - // Len(0x4100) is 1 (one character) where LenB is 2. Calling ToString() on a byte[] yields the - // literal "System.Byte[]", which silently produced nonsense — Len returned 13 for every value. - "LEN" => Convert1(f, v => ToText(v).Length), - "LCASE" => Convert1(f, v => ToText(v).ToLowerInvariant()), - "UCASE" => Convert1(f, v => ToText(v).ToUpperInvariant()), - "TRIM" => Convert1(f, v => ToText(v).Trim(' ')), - "LTRIM" => Convert1(f, v => ToText(v).TrimStart(' ')), - "RTRIM" => Convert1(f, v => ToText(v).TrimEnd(' ')), + "CDEC" => DecimalArgument(f, number => number), + "CSTR" => Convert1(f, ConcatText), + "CDATE" => Convert1(f, v => ToDate(v)), + "CVAR" => Evaluate(f.Arguments[0]), + + // VBA/Access string functions (verified vs ACE). A value that is not text is read as CStr writes it + // (ConcatText): True is "-1", a date in the regional format, a GUID braced, and a binary value as + // UTF-16 text, so Len(0x4100) is 1 where LenB is 2. Positions are 1-based and read as the conversion + // functions read numbers; a Null argument gives Null. Text compares in the database sort order unless + // a compare argument of 0 asks for a binary comparison. + "LEN" => Convert1(f, v => ConcatText(v).Length), + "LCASE" => Convert1(f, v => ConcatText(v).ToLowerInvariant()), + "UCASE" => Convert1(f, v => ConcatText(v).ToUpperInvariant()), + "TRIM" => Convert1(f, v => ConcatText(v).Trim(TrimmedSpaces)), + "LTRIM" => Convert1(f, v => ConcatText(v).TrimStart(TrimmedSpaces)), + "RTRIM" => Convert1(f, v => ConcatText(v).TrimEnd(TrimmedSpaces)), "LEFT" => StringInt(f, static (s, n) => n <= 0 ? "" : n >= s.Length ? s : s[..n]), "RIGHT" => StringInt(f, static (s, n) => n <= 0 ? "" : n >= s.Length ? s : s[^n..]), "MID" => Mid(f), "INSTR" => Instr(f), "REPLACE" => Replace(f), - // Date/time functions (VBA/Access). All propagate NULL on a date argument. + // Date/time functions (verified vs ACE). A date argument is read as CDate reads it: text as a date in + // the regional format, otherwise as a number, and a number as the date at that serial. Settings and + // counts are read as CInt reads them. A Null argument gives Null. LibRed keeps milliseconds where ACE + // rounds to the second, so Second(0.00001) is 0 here and 1 in ACE. "DATEADD" => DateAdd(f), "DATEDIFF" => DateDiff(f), - "DATESERIAL" => DateParts(f, (y, m, d) => new DateTime(y, 1, 1).AddMonths(m - 1).AddDays(d - 1)), - "TIMESERIAL" => DateParts(f, (h, m, s) => DateTime.FromOADate(0).AddHours(h).AddMinutes(m).AddSeconds(s)), + "DATESERIAL" => DateParts(f, DateSerial), + "TIMESERIAL" => DateParts(f, static (h, m, s) => OaDate((h * 3600 + m * 60 + s) / 86400.0)), "NOW" => DateTime.Now, "DATE" => DateTime.Today, "TIME" => DateTime.FromOADate(0).Add(DateTime.Now.TimeOfDay), - "YEAR" => DatePartOf(f, d => d.Year), - "MONTH" => DatePartOf(f, d => d.Month), - "DAY" => DatePartOf(f, d => d.Day), - "HOUR" => DatePartOf(f, d => d.Hour), - "MINUTE" => DatePartOf(f, d => d.Minute), - "SECOND" => DatePartOf(f, d => d.Second), - "WEEKDAY" => DatePartOf(f, d => (int)d.DayOfWeek + 1), // Access: Sunday = 1 - // DateValue = the date at midnight; TimeValue = the time on the Jet epoch (1899-12-30) — both - // NULL-propagating and verified against ACE. IsDate is a predicate (true only for a date or a - // date/time-parseable string; a number, NULL or unparseable string is false — verified vs ACE). - "DATEVALUE" => Convert1(f, v => ((DateTime)ToDate(v)).Date), - "TIMEVALUE" => Convert1(f, v => DateTime.FromOADate(0).Add(((DateTime)ToDate(v)).TimeOfDay)), - "ISDATE" => IsDateValue(Evaluate(f.Arguments[0])), + "YEAR" => Convert1(f, v => ToDate(v).Year), + "MONTH" => Convert1(f, v => ToDate(v).Month), + "DAY" => Convert1(f, v => ToDate(v).Day), + "HOUR" => Convert1(f, v => ToDate(v).Hour), + "MINUTE" => Convert1(f, v => ToDate(v).Minute), + "SECOND" => Convert1(f, v => ToDate(v).Second), + "WEEKDAY" => Weekday(f), + // DateValue is the date at midnight and TimeValue the time on 1899-12-30. Unlike CDate they take only a + // date or text that reads as one; a number is a type mismatch. IsDate is true for exactly those. + "DATEVALUE" => Convert1(f, v => DateValueArgument(v).Date), + "TIMEVALUE" => Convert1(f, v => DateTime.FromOADate(0).Add(DateValueArgument(v).TimeOfDay)), + "ISDATE" => TryDateText(Evaluate(f.Arguments[0]), out _), // Jet VBA math functions (double precision). SQR = sqrt, ATN = atan, LOG = natural log. - // Acos/Asin/Atan2/Floor/Ceiling/Log10/Log-base are emitted by EF as expressions built from - // these plus arithmetic, so they need no dedicated cases. - "SIN" => UnaryDouble(f, Math.Sin), - "COS" => UnaryDouble(f, Math.Cos), - "TAN" => UnaryDouble(f, Math.Tan), + // ACE's Tan is its Sin over its Cos, which differs from a direct tangent in the last digit. + "SIN" => UnaryDouble(f, Trigonometric(Math.Sin)), + "COS" => UnaryDouble(f, Trigonometric(Math.Cos)), + "TAN" => UnaryDouble(f, Trigonometric(x => Math.Sin(x) / Math.Cos(x))), "ATN" => UnaryDouble(f, Math.Atan), "EXP" => UnaryDouble(f, Math.Exp), - "LOG" => UnaryDouble(f, Math.Log), + // Log(base, x), the standard's two-argument form, is a LibRed extension beside Access's natural log. + "LOG" => f.Arguments.Count == 2 + ? BinaryDouble(f, static (b, x) => b <= 0 || b == 1 || x <= 0 ? double.NaN : Math.Log(x, b)) + : UnaryDouble(f, Math.Log), "SQR" => UnaryDouble(f, Math.Sqrt), // SGN sits apart from the group above: it takes a double but yields an Integer, both in VBA // (Sgn returns Variant/Integer) and in .NET (Math.Sign returns int). Going through UnaryDouble // widened that int straight back to a double, which only showed once EF projected the value // instead of comparing it - GetInt32 on a boxed Double throws. - "SGN" => Convert1(f, v => Math.Sign(Convert.ToDouble(v, CultureInfo.InvariantCulture))), + "SGN" => Convert1(f, v => Math.Sign(Dbl(ConversionNumber(v)))), + // The standard SQL math functions, LibRed extensions (ACE has none of them). The ones Access has under + // another name are that function - Floor is Int, Sqrt is Sqr, Ln is Log, Atan is Atn, Sign is Sgn, and + // Power is the ^ operator - so they read their arguments and fail as it does. + "FLOOR" => Numeric1(f, Math.Floor, Math.Floor, keepsDate: true), + "CEILING" or "CEIL" => Numeric1(f, Math.Ceiling, Math.Ceiling, keepsDate: true), + "SIGN" => Convert1(f, v => Math.Sign(Dbl(ConversionNumber(v)))), + "SQRT" => UnaryDouble(f, Math.Sqrt), + "LN" => UnaryDouble(f, Math.Log), + "LOG10" => UnaryDouble(f, Math.Log10), + "POWER" => Evaluate(new BinaryExpression(BinaryOperator.Power, f.Arguments[0], f.Arguments[1])), + "ASIN" => UnaryDouble(f, Math.Asin), + "ACOS" => UnaryDouble(f, Math.Acos), + "ATAN" => UnaryDouble(f, Math.Atan), + "ATAN2" => BinaryDouble(f, Math.Atan2), + "SINH" => UnaryDouble(f, Math.Sinh), + "COSH" => UnaryDouble(f, Math.Cosh), + "TANH" => UnaryDouble(f, Math.Tanh), + "DEGREES" => UnaryDouble(f, x => x * 180 / Math.PI), + "RADIANS" => UnaryDouble(f, x => x * Math.PI / 180), + "PI" => Math.PI, // More VBA/Access built-ins (verified vs ACE via the function-whitelist sweep). All NULL-propagating // via Convert1 unless noted; positions are 1-based. - "ASC" => Convert1(f, v => (int)v.ToString()![0]), - "CHR" => Convert1(f, v => ((char)Convert.ToInt32(v, CultureInfo.InvariantCulture)).ToString()), - "SPACE" => Convert1(f, v => new string(' ', Convert.ToInt32(v, CultureInfo.InvariantCulture))), + // Asc and Chr work in the system ANSI code page (Chr takes 0-255; Chr(128) is '€', Asc('Ā') is 65 by + // best fit); AscW and ChrW in UTF-16 code units, AscW signed and ChrW taking -32768 to 65535. + "ASC" => Convert1(f, v => (int)Ansi.GetBytes(FirstCharacter(v))[0]), + "CHR" => Convert1(f, v => AnsiCharacter(InRange(AsLong(v), 0, 255)).ToString()), + "SPACE" => Convert1(f, v => new string(' ', Count(v))), "STRING" => StringOf(f), // String(count, char) → char repeated count times - "STRREVERSE" => Convert1(f, v => new string(v.ToString()!.Reverse().ToArray())), + "STRREVERSE" => Convert1(f, v => new string(ConcatText(v).Reverse().ToArray())), "STRCOMP" => StrComp(f), // -1/0/1 (case-insensitive, Access "Compare Database") "STR" => Convert1(f, VbaStr), // number → text with a leading space when non-negative - "VAL" => Convert1(f, v => VbaVal(v.ToString()!)),// parse the leading numeric portion (Double), else 0 - "HEX" => Convert1(f, v => Convert.ToString(Convert.ToInt64(v, CultureInfo.InvariantCulture), 16).ToUpperInvariant()), - "OCT" => Convert1(f, v => Convert.ToString(Convert.ToInt64(v, CultureInfo.InvariantCulture), 8)), + "VAL" => Convert1(f, v => VbaVal(ConcatText(v))), + "HEX" => Convert1(f, v => RadixText(v, 16)), + "OCT" => Convert1(f, v => RadixText(v, 8)), "INSTRREV" => InstrRev(f), // last occurrence, 1-based (0 if none) // MonthName(month, [abbreviate]). The second argument was accepted by the arity table and then // ignored by Convert1, so MonthName(1, True) returned "January" where ACE returns "Jan" — a @@ -272,44 +342,44 @@ private static bool TryNiladicFunction(ColumnReference c, out object? value) "MONTHNAME" => MonthNameOf(f), "TIMER" => (DateTime.Now - DateTime.Today).TotalSeconds, "RND" => Rnd(f), - // Predicates / type inspection. IsError is always false — LibRed has no error-value type. ISNULL - // returns a Boolean (ACE reports it as -1/0); both print as a boolean here. + // Predicates / type inspection. IsError is always false — LibRed has no error-value type — but its argument + // is still evaluated, so an error in it is raised (verified vs ACE: IsError(1/0) fails). ISNULL returns a + // Boolean (ACE reports it as -1/0); both print as a boolean here. "ISNULL" => Evaluate(f.Arguments[0]) is null, "ISNUMERIC" => IsNumericValue(Evaluate(f.Arguments[0])), - "ISERROR" => false, - "TYPENAME" => TypeNameOf(Evaluate(f.Arguments[0])), - "VARTYPE" => VarTypeOf(Evaluate(f.Arguments[0])), + "ISERROR" => IsError(f), + "TYPENAME" => TypeNameOf(Evaluate(f.Arguments[0]), f.Arguments[0]), + "VARTYPE" => VarTypeOf(Evaluate(f.Arguments[0]), f.Arguments[0]), "STRCONV" => StrConv(f), "WEEKDAYNAME" => WeekdayNameOf(f), "PARTITION" => PartitionOf(f), "FORMAT" => FormatValue(f), - "FORMATCURRENCY" => Convert.ToDecimal(FinArg(f, 0), CultureInfo.InvariantCulture).ToString("C" + FmtDigits(f, 1), CultureInfo.CurrentCulture), - "FORMATNUMBER" => Convert.ToDouble(FinArg(f, 0), CultureInfo.InvariantCulture).ToString("N" + FmtDigits(f, 1), CultureInfo.CurrentCulture), - "FORMATPERCENT" => FinArg(f, 0).ToString(FmtDigits(f, 1) > 0 ? "0." + new string('0', FmtDigits(f, 1)) + "%" : "0%", CultureInfo.CurrentCulture), - "FORMATDATETIME" => FormatDateTimeFn(f), - "RGB" => (int)(FinArg(f, 0) % 256) + ((int)(FinArg(f, 1) % 256) << 8) + ((int)(FinArg(f, 2) % 256) << 16), - "QBCOLOR" => QbColor((int)FinArg(f, 0)), + "FORMATCURRENCY" => FormatStyled(f, NumberStyle.Currency), + "FORMATNUMBER" => FormatStyled(f, NumberStyle.Number), + "FORMATPERCENT" => FormatStyled(f, NumberStyle.Percent), + "FORMATDATETIME" => FormatDateTime(f), + "RGB" => Rgb(f), + "QBCOLOR" => Convert1(f, v => QbColors[Setting(v, 0, 15)]), // Financial functions (verified vs ACE). rate is per period; pv/fv/pmt sign conventions follow VBA. - "PMT" => Pmt(f), - "FV" => Fv(f), - "PV" => Pv(f), - "NPER" => NPer(f), - "IPMT" => IPmt(f), - "PPMT" => PPmt(f), - "RATE" => Rate(f), - "SLN" => (FinArg(f, 0) - FinArg(f, 1)) / FinArg(f, 2), - "SYD" => (FinArg(f, 0) - FinArg(f, 1)) * (FinArg(f, 2) - FinArg(f, 3) + 1) / (FinArg(f, 2) * (FinArg(f, 2) + 1) / 2), - "DDB" => Ddb(f), - - // Wide (Unicode code-point) variants. AscW = the first char's code point; ChrW = the char for a code - // point (unlike Chr, not restricted to a byte). Verified vs ACE: ChrW(233) → 'é'. - "ASCW" => Convert1(f, v => (int)ToText(v)[0]), - "CHRW" => Convert1(f, v => ((char)Convert.ToInt32(v, CultureInfo.InvariantCulture)).ToString()), + "PMT" => Financial(f, a => Pmt(a[0], a[1], a[2], a[3], Due(a[4]))), + "FV" => Financial(f, a => Fv(a[0], a[1], a[2], a[3], Due(a[4]))), + "PV" => Financial(f, a => Pv(a[0], a[1], a[2], a[3], Due(a[4]))), + "NPER" => Financial(f, a => NPer(a[0], a[1], a[2], a[3], Due(a[4]))), + "IPMT" => Financial(f, a => IPmt(a[0], a[1], a[2], a[3], a[4], Due(a[5]))), + "PPMT" => Financial(f, a => PPmt(a[0], a[1], a[2], a[3], a[4], Due(a[5]))), + "RATE" => Financial(f, a => Rate(a[0], a[1], a[2], a[3], Due(a[4]), f.Arguments.Count > 5 ? a[5] : 0.1)), + "SLN" => Financial(f, a => Sln(a[0], a[1], a[2])), + "SYD" => Financial(f, a => Syd(a[0], a[1], a[2], a[3])), + "DDB" => Financial(f, a => Ddb(a[0], a[1], a[2], a[3], f.Arguments.Count > 4 ? a[4] : 2)), + + "ASCW" => Convert1(f, v => (int)(short)FirstCharacter(v)[0]), + "CHRW" => Convert1(f, v => ((char)(InRange(AsLong(v), -32768, 65535) & 0xFFFF)).ToString()), // Byte variants operate on the UTF-16 byte layout (2 bytes/char): LenB = 2×length, AscB = the low // byte of the first char, and Left/Right/Mid/InStr count bytes. Verified vs ACE (LenB('abc')=6, // InStrB(1,'abc','b')=3). ChrB is intentionally absent — ACE's expression service has no ChrB. - "ASCB" => Convert1(f, v => (int)ToBytes(v)[0]), + "ASCB" => Convert1(f, v => (int)ToBytes(FirstCharacter(v))[0]), "LENB" => Convert1(f, v => ToBytes(v).Length), + "DATALENGTH" => Convert1(f, v => DataLength(v, IsCurrency(f.Arguments[0]))), "LEFTB" => ByteLeft(f), "RIGHTB" => ByteRight(f), "MIDB" => ByteMid(f), @@ -337,15 +407,17 @@ internal static void ValidateArity(string name, int count) (int Min, int Max)? range = name switch { // Conversion, unary numeric/string/date/inspection functions and single-argument aliases. - "CBOOL" or "CBYTE" or "CINT" or "CLNG" or "CSNG" or "CDBL" or "CCUR" or "CDEC" + "CBOOL" or "CBYTE" or "CINT" or "CLNG" or "CLNGLNG" or "CSNG" or "CDBL" or "CCUR" or "CDEC" or "CSTR" or "CDATE" or "CVAR" - or "ABS" or "SGN" or "INT" or "FIX" or "SQR" or "EXP" or "LOG" or "SIN" or "COS" + or "ABS" or "SGN" or "INT" or "FIX" or "SQR" or "EXP" or "SIN" or "COS" or "TAN" or "ATN" + or "FLOOR" or "CEILING" or "CEIL" or "SIGN" or "SQRT" or "LN" or "LOG10" or "ASIN" or "ACOS" + or "ATAN" or "SINH" or "COSH" or "TANH" or "DEGREES" or "RADIANS" or "LEN" or "LCASE" or "UCASE" or "TRIM" or "LTRIM" or "RTRIM" or "SPACE" or "STRREVERSE" or "STR" or "VAL" or "CHR" or "ASC" or "HEX" or "OCT" or "DATEVALUE" or "TIMEVALUE" or "YEAR" or "MONTH" or "DAY" or "HOUR" or "MINUTE" or "SECOND" or "ISDATE" or "ISNULL" or "ISNUMERIC" or "ISERROR" or "TYPENAME" or "VARTYPE" - or "QBCOLOR" or "ASCW" or "CHRW" or "ASCB" or "LENB" => (1, 1), + or "QBCOLOR" or "ASCW" or "CHRW" or "ASCB" or "LENB" or "DATALENGTH" => (1, 1), "LEFT" or "RIGHT" or "STRING" or "LEFTB" or "RIGHTB" => (2, 2), "MID" or "MIDB" => (2, 3), @@ -359,6 +431,8 @@ internal static void ValidateArity(string name, int count) // COALESCE(expression [, ...n]). SQL Server insists on two, but one is harmless and the standard's // own grammar allows it, so only an empty list is rejected. "COALESCE" => (1, int.MaxValue), + // GREATEST/LEAST(expression [, ...n]), as SQL Server and PostgreSQL take them: one argument or more. + "GREATEST" or "LEAST" => (1, int.MaxValue), "NOW" or "DATE" or "TIME" or "TIMER" or "GENUNIQUEID" or "GENGUID" => (0, 0), "DATEADD" => (3, 3), @@ -370,6 +444,9 @@ internal static void ValidateArity(string name, int count) "RGB" => (3, 3), "ROUND" => (1, 2), + "LOG" => (1, 2), + "POWER" or "ATAN2" => (2, 2), + "PI" => (0, 0), "RND" => (0, 1), "REPLACE" => (3, 6), "FORMAT" => (1, 4), @@ -384,8 +461,10 @@ internal static void ValidateArity(string name, int count) "SYD" => (4, 4), "DDB" => (4, 5), - "COUNT" or "SUM" or "AVG" or "MIN" or "MAX" or "FIRST" or "LAST" or "STDEV" or "VAR" - or "STDEVP" or "VARP" or "STDDEV" or "STDDEVP" => (1, 1), + "FIRST" or "LAST" => (1, 1), + _ when RunningAggregate.Supports(name) => RunningAggregate.IsPair(name) ? (2, 2) : (1, 1), + // The fraction and the WITHIN GROUP key, which the parser appends; it has checked the call's shape. + "PERCENTILE_CONT" or "PERCENTILE_DISC" => (2, 2), _ => null, }; bool invalidPairs = name == "SWITCH" && count % 2 != 0; @@ -397,18 +476,17 @@ internal static void ValidateArity(string name, int count) : valid.Max == int.MaxValue ? $"at least {valid.Min}" : $"{valid.Min} to {valid.Max}") + ")."); } - /// Access Choose(index, choice-1, choice-2, …): returns the 1-based choice at - /// 's index, or NULL when the index is out of range (verified vs ACE: Choose(0,…) - /// and Choose(5,…) on three choices both return Null). A NULL index is an error in ACE ("Data type - /// mismatch"). Only the selected choice is evaluated. + /// Access Choose(index, choice-1, choice-2, …) (verified vs ACE): the choice at the index, read as + /// a number and truncated (1.5 and 1.9 are 1, True is -1), or Null when there is no such choice. Every choice is + /// evaluated, so an error in any of them is raised. A Null index is a type mismatch. private object? Choose(FunctionCall f) { object? indexValue = Evaluate(f.Arguments[0]); if (indexValue is null) throw new InvalidOperationException("Data type mismatch in criteria expression: Choose() index is null."); - int index = Convert.ToInt32(indexValue, CultureInfo.InvariantCulture); - int choiceCount = f.Arguments.Count - 1; - return index < 1 || index > choiceCount ? null : Evaluate(f.Arguments[index]); + double index = Math.Truncate(Dbl(ConversionNumber(indexValue))); + object?[] choices = f.Arguments.Skip(1).Select(Evaluate).ToArray(); + return index < 1 || index > choices.Length ? null : choices[(int)index - 1]; } /// @@ -461,347 +539,469 @@ internal static void ValidateArity(string name, int count) return null; } - /// Access Switch(cond-1, value-1, cond-2, value-2, …): evaluates the conditions left to - /// right and returns the value paired with the first true one, or NULL if none is true (verified vs ACE). - /// The argument count must be even (condition/value pairs) — an odd count is an error in ACE ("Wrong number - /// of arguments"). Only the matched value is evaluated. + /// + /// GREATEST(a, b, …) and LEAST(a, b, …) — the largest or smallest of the arguments, compared + /// as the < and > operators compare. Access/ACE has neither, so like COALESCE they are + /// reachable from LibRed's extended SQL mode and from hand-written SQL. + /// + /// + /// NULL arguments are ignored and the answer is NULL only when every argument is NULL — SQL Server's and + /// PostgreSQL's rule, and the one EF Core translates Math.Max/Math.Min and a Max()/ + /// Min() over an inline collection against. (MySQL and Oracle instead return NULL when any argument + /// is NULL.) Every argument is evaluated, each once. Of equal values the first is returned. + /// + private object? Extreme(FunctionCall f, bool greatest) + { + object? result = null; + foreach (Expression argument in f.Arguments) + { + object? value = Evaluate(argument); + if (value is null) + continue; + if (result is null || (greatest ? Compare(value, result) > 0 : Compare(value, result) < 0)) + result = value; + } + + return result; + } + + /// Access Switch(cond-1, value-1, cond-2, value-2, …) (verified vs ACE): the value paired with the + /// first true condition, or Null if none is. A condition is read as CBool reads it, so 'False' is False and text + /// that is neither a Boolean nor a number is a type mismatch; Null is False. Every argument is evaluated, so an + /// error in any of them is raised. An odd number of arguments is an error ("Wrong number of arguments"). private object? Switch(FunctionCall f) { if (f.Arguments.Count % 2 != 0) throw new InvalidOperationException( "Wrong number of arguments used with function Switch (expects condition/value pairs)."); - for (int i = 0; i < f.Arguments.Count; i += 2) - if (IsTrue(f.Arguments[i])) - return Evaluate(f.Arguments[i + 1]); + object?[] values = f.Arguments.Select(Evaluate).ToArray(); + for (int i = 0; i < values.Length; i += 2) + if (values[i] is { } condition && VbaBool(condition)) + return values[i + 1]; return null; } private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); - /// Access String(count, character): 's first arg repeated. The character - /// arg may be a string (first char used) or a character code. NULL-propagating. + /// The system ANSI code page, which Asc and Chr work in; unmappable characters take their best fit. + private static Encoding Ansi => + CodePagesEncodingProvider.Instance.GetEncoding(CultureInfo.CurrentCulture.TextInfo.ANSICodePage) ?? Encoding.Latin1; + + private static char AnsiCharacter(int code) => Ansi.GetString([(byte)code])[0]; + + /// The first character of a value as text; an empty text is an invalid procedure call. + private static string FirstCharacter(object v) + { + string text = ConcatText(v); + return text.Length > 0 + ? text[..1] + : throw new ArgumentException("Invalid procedure call: the text is empty."); + } + + /// A value read as CInt reads it: half to even, and past an Integer is an overflow. + private static int AsInteger(object v) => Convert.ToInt16(ConversionNumber(v), CultureInfo.InvariantCulture); + + /// A value read as CLng reads it: half to even, and past a Long is an overflow. + private static int AsLong(object v) => Int(ConversionNumber(v)); + + /// A value read as CLngLng reads it: as CLng, half to even, and past an Int64 an overflow. Text that + /// reads as a number is read exactly, not through a Double, so '9223372036854775807' loses no digit. + private static long AsLongLong(object v) => + v is string or char && TextAsDecimal(v.ToString()!) is decimal exact + ? Convert.ToInt64(exact, CultureInfo.InvariantCulture) + : Lng(ConversionNumber(v)); + + /// A whole-number argument, where outside - is an invalid + /// procedure call. + private static int InRange(int n, int least, int most = int.MaxValue) => + n >= least && n <= most ? n : throw new ArgumentException($"Invalid procedure call: {n} is out of range."); + + /// A count, length or position read as CLng reads it; below is an invalid + /// procedure call. + private static int Count(object v, int least = 0) => InRange(AsLong(v), least); + + /// + /// A function's argument at , read by : + /// when the call leaves it out, and null — the call gives Null — when it is Null. + /// + private T? Optional(FunctionCall f, int index, T absent, Func read) where T : struct => + f.Arguments.Count <= index ? absent + : Evaluate(f.Arguments[index]) is { } v ? read(v) + : null; + + /// A count, length or position argument (), or null when the argument is Null. + private int? CountArgument(FunctionCall f, int index, int least = 0, int absent = 0) => + Optional(f, index, absent, v => Count(v, least)); + + /// + /// Whether a compare argument asks for a binary comparison: 0 is binary, and 1 (the default when the argument is + /// absent) or a locale ID is textual. Anything else is an invalid procedure call — the documented + /// vbUseCompareOption (-1) and database comparison (2) included (verified vs ACE: 1031, 1033 and the neutral IDs + /// 3 and 4 are accepted, 20000 is not). Null when the argument is Null. + /// ACE compares text under a locale ID with that locale's Windows rules; LibRed uses the database order for + /// every locale. They were measured to agree on case, accents and hyphens, and to differ only in whether ß + /// matches ss: ACE expands it for 1031, 1033, 1036, 2057 and 3082 but not for 1027, 1041 or the neutral IDs. + /// + private bool? BinaryCompare(FunctionCall f, int index) => + Optional(f, index, false, v => AsLong(v) switch + { + 0 => true, + 1 => false, + var mode when mode > 2 && IsLocaleId(mode) => false, + var mode => throw new ArgumentException($"Invalid procedure call: {mode} is not a comparison."), + }); + + private static bool IsLocaleId(int lcid) + { + try + { + return lcid > 0 && CultureInfo.GetCultureInfo(lcid).LCID == lcid; + } + catch (CultureNotFoundException) + { + return false; + } + } + + /// + /// The first match of in at or after , as + /// a position and length, or (-1, 0). A textual match compares in the database sort order, so 'SS' finds 'ß' and + /// the matched length can differ from 's. + /// + private static (int Index, int Length) FindText(string text, string find, int start, bool binary) + { + if (binary || IsPlainText(text) && IsPlainText(find)) + { + int index = text.IndexOf(find, start, binary ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase); + return (index, find.Length); + } + int shortest = Math.Max(1, (find.Length + 1) / 2); + for (int i = start; i < text.Length; i++) + { + for (int length = shortest; length <= Math.Min(text.Length - i, find.Length * 2); length++) + { + if (CompareText(text.Substring(i, length), find) == 0) + return (i, length); + } + } + return (-1, 0); + } + + /// Text whose database order is plain case-insensitive order: ASCII with no hyphen or apostrophe, + /// which the order weighs apart, and no trailing space, which it ignores. + private static bool IsPlainText(string text) => + text.All(c => c < 0x80 && c is not ('-' or '\'')) && !text.EndsWith(' '); + + /// + /// Access String(count, character): the character repeated. A text gives its first character (an empty + /// one is an invalid procedure call); a number is a character code in the ANSI code page, taken modulo 256 (verified + /// vs ACE: String(3, 321) is 'AAA', String(3, True) 'ÿÿÿ'). + /// private object? StringOf(FunctionCall f) { - object? countValue = Evaluate(f.Arguments[0]); - object? charValue = Evaluate(f.Arguments[1]); - if (countValue is null || charValue is null) return null; - int count = Convert.ToInt32(countValue, CultureInfo.InvariantCulture); - char ch = charValue is string s - ? (s.Length > 0 ? s[0] : ' ') - : (char)Convert.ToInt32(charValue, CultureInfo.InvariantCulture); + if (CountArgument(f, 0) is not { } count || Evaluate(f.Arguments[1]) is not { } charValue) + return null; + char ch = charValue is string or Guid or byte[] + ? FirstCharacter(charValue)[0] + : AnsiCharacter(AsLong(charValue) & 0xFF); return new string(ch, count); } - /// Access StrComp(a, b, [compare]): -1/0/1 (NULL-propagating). compare=0 is a binary - /// (case-sensitive) comparison; the default and every other mode are textual (case-insensitive, Access - /// "Option Compare Database" = Text). Verified vs ACE (StrComp('a','A',0)=1, default=0). + /// + /// Access StrComp(a, b, [compare]): -1, 0 or 1. A binary comparison orders the UTF-16 code units; a textual + /// one the database sort order, with trailing spaces breaking a tie (verified vs ACE: StrComp('ß', 'ss') is 0, + /// StrComp('a-b', 'ab') 1, StrComp('a', 'a ') -1). + /// private object? StrComp(FunctionCall f) { object? a = Evaluate(f.Arguments[0]); object? b = Evaluate(f.Arguments[1]); - if (a is null || b is null) return null; - StringComparison cmp = f.Arguments.Count > 2 - && Convert.ToInt32(Evaluate(f.Arguments[2]), CultureInfo.InvariantCulture) == 0 - ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - return Math.Sign(string.Compare(a.ToString(), b.ToString(), cmp)); + if (a is null || b is null || BinaryCompare(f, 2) is not { } binary) + return null; + string left = ConcatText(a), right = ConcatText(b); + if (binary) + return Math.Sign(string.CompareOrdinal(left, right)); + int order = CompareText(left, right); + return order != 0 ? order : Math.Sign(TrailingSpaces(left) - TrailingSpaces(right)); + + static int TrailingSpaces(string s) => s.Length - s.TrimEnd(' ').Length; } /// Access InStrRev(string1, string2, [start=-1], [compare]): the 1-based position of the last /// occurrence of string2 in string1 (0 if not found), searching within the first start characters /// (the match must end at or before start; start=-1 means the whole string). Case-insensitive /// unless compare=0 (binary). Semantics verified vs ACE, including its quirks: an empty needle returns the - /// effective start position; start=0 (or <-1) → "Invalid procedure call"; and — unlike - /// InStr — a NULL argument raises "Data type mismatch" rather than propagating NULL. + /// effective start position; start=0 (or <-1) → "Invalid procedure call", and one past the end of + /// string1 gives 0; and — unlike InStr — a NULL string raises "Data type mismatch" rather than propagating + /// NULL. private object? InstrRev(FunctionCall f) { object? s1v = Evaluate(f.Arguments[0]); object? s2v = Evaluate(f.Arguments[1]); if (s1v is null || s2v is null) throw new InvalidOperationException("Data type mismatch in criteria expression: InStrRev() argument is null."); - string s1 = s1v.ToString()!, s2 = s2v.ToString()!; + string s1 = ConcatText(s1v), s2 = ConcatText(s2v); - int start = f.Arguments.Count > 2 ? Convert.ToInt32(Evaluate(f.Arguments[2]), CultureInfo.InvariantCulture) : -1; - if (start == -1) start = s1.Length; - else if (start < 1) - throw new InvalidOperationException("Invalid procedure call: InStrRev() start must be -1 or a positive position."); - - StringComparison cmp = f.Arguments.Count > 3 - && Convert.ToInt32(Evaluate(f.Arguments[3]), CultureInfo.InvariantCulture) == 0 - ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + if (CountArgument(f, 2, least: -1, absent: -1) is not { } given) + return null; + if (given == 0) + throw new ArgumentException("Invalid procedure call: InStrRev() start must be -1 or a positive position."); + if (given > s1.Length) + return 0; + if (BinaryCompare(f, 3) is not { } binary) + return null; + int start = given > 0 ? given : s1.Length; if (s1.Length == 0) return 0; - int window = Math.Min(start, s1.Length); // search within Left(string1, start) - if (s2.Length == 0) return window; // empty needle → the effective start position - int idx = s1[..window].LastIndexOf(s2, cmp); - return idx < 0 ? 0 : idx + 1; + string window = s1[..start]; // search within Left(string1, start) + if (s2.Length == 0) return start; // empty needle → the effective start position + int last = -1; + for ((int index, int _) = FindText(window, s2, 0, binary); index >= 0; (index, _) = FindText(window, s2, index + 1, binary)) + last = index; + return last + 1; } - /// VBA Str(number): the number as text, with a leading space for non-negative values (VBA - /// reserves that column for the sign). + /// + /// Access Str(number) (verified vs ACE): the number read as the conversion functions read it and written + /// as CStr writes it, but always with a period and no zero before it, and with a space where a positive number's + /// sign would go (Str(0.5) is " .5", Str(True) is "-1"). A date is written as CStr writes it. + /// private static string VbaStr(object v) { - double d = Convert.ToDouble(v, CultureInfo.InvariantCulture); - string s = d.ToString(CultureInfo.InvariantCulture); - return d >= 0 ? " " + s : s; + if (v is DateTime date) + return ConcatText(date); + NumberFormatInfo invariant = NumberFormatInfo.InvariantInfo; + string text = NumericOperand(v)! switch + { + bool b => b ? "-1" : "0", + double d => FloatingText(d, 15, invariant), + float f => FloatingText(f, 7, invariant), + decimal m => m.ToString("0.############################", invariant), + var n => Convert.ToString(n, invariant)!, + }; + if (text.StartsWith("0.", StringComparison.Ordinal)) + text = text[1..]; + else if (text.StartsWith("-0.", StringComparison.Ordinal)) + text = "-" + text[2..]; + return text[0] == '-' ? text : " " + text; } - /// VBA Val(string): the leading number as a Double (0 if none). VBA first strips ALL - /// whitespace (so "3 .1 4" → 3.14, " - 5" → -5), then reads the leading number — recognising - /// &H hex and &O octal prefixes — and stops at the first character it can't use - /// (verified vs ACE). - private static object VbaVal(string s) + /// + /// Access Val(string) (verified vs ACE): the number at the start of the text, as a Double, or 0. Spaces, + /// tabs, carriage returns and line feeds are removed first, wherever they are (other whitespace is not). The + /// number may have a sign, a decimal point and an e or d exponent. &H hex and + /// &O octal take no sign and keep their low 32 bits, read as an Integer when they fit 16 bits + /// (&HFFFF is -1) and otherwise as a Long. A number past a Double is an overflow. + /// + private static double VbaVal(string s) { - string t = new string(s.Where(c => !char.IsWhiteSpace(c)).ToArray()); - if (t.Length == 0) return 0.0; - - Match hex = Regex.Match(t, @"^([+-]?)&[Hh]([0-9A-Fa-f]+)"); - if (hex.Success) + string t = string.Concat(s.Where(c => c is not (' ' or '\t' or '\r' or '\n'))); + if (t.Length > 1 && t[0] == '&' && (char.ToUpperInvariant(t[1]) is var prefix && prefix is 'H' or 'O')) { - long h = Convert.ToInt64(hex.Groups[2].Value, 16); - return (double)(hex.Groups[1].Value == "-" ? -h : h); + int radix = prefix == 'H' ? 16 : 8; + ulong bits = 0; + foreach (char c in t[2..]) + { + int digit = char.IsAsciiDigit(c) ? c - '0' : char.IsAsciiHexDigit(c) ? char.ToUpperInvariant(c) - 'A' + 10 : radix; + if (digit >= radix) + break; + bits = unchecked(bits * (ulong)radix + (ulong)digit); + } + bits &= 0xFFFF_FFFF; + return bits <= 0xFFFF ? (short)bits : (int)bits; } - Match oct = Regex.Match(t, @"^([+-]?)&[Oo]([0-7]+)"); - if (oct.Success) + + Match number = Regex.Match(t, @"^[+-]?(\d+\.?\d*|\.\d+)([eEdD][+-]?\d+)?"); + if (!number.Success) + return 0; + double value = double.Parse(number.Value.Replace('d', 'e').Replace('D', 'e'), NumberStyles.Float, CultureInfo.InvariantCulture); + return double.IsFinite(value) ? value : throw new OverflowException($"Overflow: '{s}' is too large for a number."); + } + + /// + /// Access Hex and Oct (verified vs ACE): the bits of the number read as the conversion functions + /// read it — 16 of them for an Integer or a Boolean (Hex(True) is FFFF), 32 for a Long, and otherwise the + /// value rounded half to even as a 64-bit whole number (Hex(-1.5) is FFFFFFFFFFFFFFFE). + /// + private static string RadixText(object v, int radix) + { + long bits = NumericOperand(v)! switch { - long o = Convert.ToInt64(oct.Groups[2].Value, 8); - return (double)(oct.Groups[1].Value == "-" ? -o : o); - } - Match dec = Regex.Match(t, @"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?"); - return dec.Success && double.TryParse(dec.Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double d) - ? d : 0.0; + bool b => b ? 0xFFFF : 0, + short s => (ushort)s, + int i => (uint)i, + var n => Lng(Serial(n)), + }; + return Convert.ToString(bits, radix).ToUpperInvariant(); + } + + private bool IsError(FunctionCall f) + { + Evaluate(f.Arguments[0]); + return false; } - /// VBA IsNumeric(value): true for a number or a numeric string. + /// Access IsNumeric(value) (verified vs ACE): true for a number or a Boolean, and for text that + /// reads as a number the way + reads it ('$5', '&HFF', '(1)' and '1d2' do; '1e400' does not). A date, a + /// GUID, a binary value and Null are not numeric. private static bool IsNumericValue(object? v) => v switch { - null or bool => false, - byte or short or int or long or float or double or decimal => true, - _ => double.TryParse(v.ToString(), NumberStyles.Any, CultureInfo.InvariantCulture, out _), + null or DateTime or Guid or byte[] => false, + string or char => TryTextAsNumber(v.ToString()!) is not null, + _ => true, }; - /// VBA TypeName(value) — the Access type name (verified vs ACE, e.g. an Int32 literal → "Long"). - private static string TypeNameOf(object? v) => v switch + /// VBA TypeName(value) — the Access type name of the value's type (e.g. an Int32 literal → "Long"). + /// A Boolean is "Boolean" and a Byte "Byte", as LibRed holds them; ACE reports both as integers. Currency and + /// Decimal share , so which one a value is comes from its . + private string TypeNameOf(object? v, Expression expression) => v switch { + decimal when IsCurrency(expression) => "Currency", + decimal => "Decimal", null => "Null", bool => "Boolean", byte => "Byte", short => "Integer", - int or long => "Long", + int => "Long", + long or ulong => "LongLong", float => "Single", double => "Double", - decimal => "Currency", DateTime => "Date", string => "String", // A binary column reports as String, not Byte[] — ACE's expression service sees the value as a // UTF-16 string (VarType 8 = VT_BSTR), so TypeName must say so even though LibRed holds a byte[]. - byte[] => "String", + byte[] or Guid or char => "String", _ => v.GetType().Name, }; - /// VBA VarType(value) — the Access variant type code (vbLong=3, vbString=8, …). - private static int VarTypeOf(object? v) => v switch + /// VBA VarType(value) — the Access variant type code (vbLong=3, vbString=8, …), with the same + /// mapping as . + private int VarTypeOf(object? v, Expression expression) => v switch { + decimal when IsCurrency(expression) => 6, // vbCurrency + decimal => 14, // vbDecimal null => 1, // vbNull bool => 11, // vbBoolean byte => 17, // vbByte short => 2, // vbInteger - int or long => 3, // vbLong + int => 3, // vbLong + long or ulong => 20, // vbLongLong float => 4, // vbSingle double => 5, // vbDouble - decimal => 6, // vbCurrency DateTime => 7, // vbDate _ => 8, // vbString }; - /// Access Format(value[, format]). Named formats (Currency, Percent, Short Date, …) and the - /// custom numeric/date/string format strings, driven off — as ACE - /// drives them off the OS regional settings (so date/currency output is locale-dependent, matching ACE on a - /// given host). Custom date formats translate VBA tokens to .NET (VBA mm=month/nn=minutes/ - /// hh=hour, plus q=quarter). NULL-propagating; no/empty format → the default string. - private object? FormatValue(FunctionCall f) - { - object? value = Evaluate(f.Arguments[0]); - if (value is null) return null; - string? fmt = f.Arguments.Count > 1 ? Evaluate(f.Arguments[1])?.ToString() : null; - if (string.IsNullOrEmpty(fmt)) return value.ToString(); - - CultureInfo c = CultureInfo.CurrentCulture; - // Named formats (case-insensitive). Numeric/boolean names first, then date/time names. - switch (fmt.Trim().ToLowerInvariant()) - { - case "general number": return Convert.ToDecimal(value, CultureInfo.InvariantCulture).ToString(c); - case "currency": return Convert.ToDecimal(value, CultureInfo.InvariantCulture).ToString("C", c); - case "fixed": return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("0.00", c); - case "standard": return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("#,##0.00", c); - case "percent": return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("0.00%", c); - case "scientific": return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("0.00E+00", c); - case "yes/no": return IsZeroValue(value) ? "No" : "Yes"; - case "true/false": return IsZeroValue(value) ? "False" : "True"; - case "on/off": return IsZeroValue(value) ? "Off" : "On"; - case "general date": return ((DateTime)ToDate(value)).ToString(c); - case "long date": return ((DateTime)ToDate(value)).ToString("D", c); - case "medium date": return ((DateTime)ToDate(value)).ToString("dd-MMM-yy", c); - case "short date": return ((DateTime)ToDate(value)).ToString("d", c); - case "long time": return ((DateTime)ToDate(value)).ToString("T", c); - case "medium time": return ((DateTime)ToDate(value)).ToString("hh:mm tt", c); - case "short time": return ((DateTime)ToDate(value)).ToString("HH:mm", c); - } - - // Custom format strings. '0'/'#' → numeric (VBA numeric tokens map ~directly to .NET). Otherwise a date - // token letter → date format (VBA→.NET translation). Otherwise a string format ('>' upper, '<' lower). - if (fmt.IndexOfAny(['0', '#']) >= 0) - return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString(fmt, c); - if (fmt.IndexOfAny(['y', 'Y', 'm', 'M', 'd', 'D', 'h', 'H', 'n', 'N', 's', 'S', 'q', 'Q']) >= 0) - { - DateTime dt = (DateTime)ToDate(value); - return dt.ToString(TranslateVbaDateFormat(fmt, dt), c); - } - return fmt switch - { - ">" => value.ToString()!.ToUpperInvariant(), - "<" => value.ToString()!.ToLowerInvariant(), - _ => value.ToString(), - }; - } - - /// True when a value is zero/false — for the Yes/No, True/False, On/Off named formats. - private static bool IsZeroValue(object v) => - v is bool b ? !b : Convert.ToDouble(v, CultureInfo.InvariantCulture) == 0; - - /// Translates a VBA date/time format string into the equivalent .NET custom format. VBA differs from - /// .NET on m=month (vs minutes), n=minutes, and h=24-hour unless AM/PM is present; and it - /// has q=quarter, which .NET lacks (emitted as an escaped literal digit). - private static string TranslateVbaDateFormat(string vba, DateTime dt) - { - bool twelveHour = vba.Contains("am/pm", StringComparison.OrdinalIgnoreCase) - || vba.Contains("a/p", StringComparison.OrdinalIgnoreCase); - var sb = new StringBuilder(); - for (int i = 0; i < vba.Length;) - { - char ch = vba[i]; - // AM/PM tokens. - if (i + 4 < vba.Length + 1 && vba.AsSpan(i).StartsWith("am/pm", StringComparison.OrdinalIgnoreCase)) - { sb.Append("tt"); i += 5; continue; } - if (ch is '\\' && i + 1 < vba.Length) { sb.Append('\\').Append(vba[i + 1]); i += 2; continue; } - if (ch is '"') - { - int j = i + 1; - while (j < vba.Length && vba[j] != '"') { sb.Append('\\').Append(vba[j]); j++; } - i = j + 1; continue; - } - char lower = char.ToLowerInvariant(ch); - if ("ymdhnsq".IndexOf(lower) >= 0) - { - int j = i; while (j < vba.Length && char.ToLowerInvariant(vba[j]) == lower) j++; - int len = j - i; - sb.Append(lower switch - { - 'y' => new string('y', len is 2 ? 2 : 4), - 'm' => new string('M', len), // VBA m/mm/mmm/mmmm = month - 'n' => new string('m', Math.Min(len, 2)), // VBA n/nn = minutes - 's' => new string('s', Math.Min(len, 2)), - 'd' => new string('d', len), - 'h' => new string(twelveHour ? 'h' : 'H', Math.Min(len, 2)), - 'q' => "\\" + ((dt.Month - 1) / 3 + 1), // quarter as an escaped literal digit - _ => new string(lower, len), - }); - i = j; continue; - } - sb.Append(ch); i++; - } - return sb.ToString(); - } + /// Whether an expression is a Currency: a Currency column, CCur, or arithmetic that keeps one. + private bool IsCurrency(Expression expression) => + NumberTypeOf(expression, scope.AllColumns(), _ => null).Class == NumberClass.Currency; - /// Access StrConv(string, conversion) (verified vs ACE): 1 = UpperCase, 2 = LowerCase, - /// 3 = ProperCase (title case); 64 = vbUnicode (reinterpret the UTF-16 bytes as one char each — doubles the - /// length with null chars); 128 = vbFromUnicode (combine char pairs into single code units). The narrow/wide - /// and Japanese Kana modes (4/16/32) raise "Invalid procedure call", matching ACE. NULL-propagating. + /// + /// Access StrConv(string, conversion, [LCID]) (verified vs ACE). 0 leaves the text as it is. 1, 2 and 3 + /// work in the ANSI code page, so a character it lacks becomes its best fit or '?': 1 is upper case, 2 lower case, + /// and 3 upper-cases the first letter and each letter after a space, tab, line break, form feed or NUL and + /// lower-cases the rest. 64 (vbUnicode) reads the text's UTF-16 bytes as ANSI characters, and 128 (vbFromUnicode) + /// packs the text's ANSI bytes two to a character. The East Asian conversions (4 to 32) and other combinations are + /// an invalid procedure call, but give Null for Null text; a conversion outside 0-255, or an LCID that is not a + /// locale, is an invalid procedure call even then. ACE keeps an odd trailing byte from 128 inside an expression + /// (LenB(StrConv("abc", 128)) is 3); LibRed text has no odd bytes, so it drops it. + /// private object? StrConv(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]); - object? modeV = Evaluate(f.Arguments[1]); - if (sv is null || modeV is null) return null; - int mode = Convert.ToInt32(modeV, CultureInfo.InvariantCulture); + if (Evaluate(f.Arguments[1]) is not { } modeV) + return null; + int mode = InRange(AsLong(modeV), 0, 255); + if (Optional(f, 2, 0, AsLong) is not { } locale) + return null; + if (locale != 0 && !IsLocaleId(locale)) + throw new ArgumentException($"Invalid procedure call: {locale} is not a locale."); + if (sv is null) + return null; + if (mode is not (0 or 1 or 2 or 3 or 64 or 128)) + throw new ArgumentException($"Invalid procedure call: {mode} is not a StrConv conversion."); // vbUnicode (64) on binary widens each byte to one Unicode char — Jet's binary→string conversion. // This is the byte-array path EF emits for `byte[].Contains(x)`: INSTR(1, STRCONV(arr, 64), 0xXX, 0). if (mode == 64 && sv is byte[] binary) return ByteArrayToString(binary); - string s = sv.ToString()!; + string s = ConcatText(sv); + Encoding ansi = Ansi; return mode switch { - 1 => s.ToUpperInvariant(), - 2 => s.ToLowerInvariant(), - 3 => EnUs.TextInfo.ToTitleCase(s.ToLowerInvariant()), - 64 => new string(Encoding.Unicode.GetBytes(s).Select(x => (char)x).ToArray()), - 128 => FromUnicodeBytes(s), - _ => throw new InvalidOperationException("Invalid procedure call: unsupported StrConv conversion mode."), + 0 => s, + 1 => ansi.GetString(ansi.GetBytes(s)).ToUpperInvariant(), + 2 => ansi.GetString(ansi.GetBytes(s)).ToLowerInvariant(), + 3 => ProperCase(ansi.GetString(ansi.GetBytes(s))), + 64 => ansi.GetString(Encoding.Unicode.GetBytes(s)), + _ => FromBytes(ansi.GetBytes(s)), }; } + private static string ProperCase(string s) + { + var chars = new char[s.Length]; + bool wordStart = true; + for (int i = 0; i < s.Length; i++) + { + chars[i] = wordStart ? char.ToUpperInvariant(s[i]) : char.ToLowerInvariant(s[i]); + wordStart = s[i] is ' ' or '\t' or '\n' or '\v' or '\f' or '\r' or '\0'; + } + return new string(chars); + } + /// Jet coerces a binary value to a string by mapping each byte to a single char (the value it /// widens back to via STRCONV(…, 64)). A byte[] reaching a string function — e.g. a /// 0xNN hex literal used as an INSTR needle — is coerced this way, not via /// ToString() (which would yield "System.Byte[]"). private static string ByteArrayToString(byte[] bytes) => new(Array.ConvertAll(bytes, b => (char)b)); - private static string ToJetString(object value) => value is byte[] b ? ByteArrayToString(b) : value.ToString()!; - - /// StrConv vbFromUnicode (128): combine successive character pairs into single UTF-16 code units - /// (low char = low byte, next char = high byte); a trailing unpaired char is dropped (verified vs ACE). - private static string FromUnicodeBytes(string s) - { - var sb = new StringBuilder(); - for (int i = 0; i + 1 < s.Length; i += 2) - sb.Append((char)(s[i] | (s[i + 1] << 8))); - return sb.ToString(); - } + private static string ToJetString(object value) => value is byte[] b ? ByteArrayToString(b) : ConcatText(value); - /// VBA WeekdayName(weekday, [abbreviate=False], [firstDayOfWeek=vbSunday]): the name of the - /// day at 1-based position weekday in a week starting from firstDayOfWeek (1=Sunday … 7=Saturday). - /// Verified vs ACE for an explicit first day (WeekdayName(1,,1)→"Sunday", (1,,2)→"Monday"). NOTE: - /// ACE's *omitted* default follows the OS regional first day; LibRed uses the VBA-documented default of - /// vbSunday for determinism, so the no-third-arg case may differ from a given ACE host. NULL-propagating. - /// Access MonthName(month, [abbreviate]) — the full English month name, or its - /// abbreviation when the second argument is true. NULL-propagating, like its Weekday sibling below. + /// Access MonthName(month, [abbreviate]): the English month name, abbreviated when asked. A month + /// outside 1-12 is an invalid procedure call (verified vs ACE, True included). private object? MonthNameOf(FunctionCall f) { - object? monthValue = Evaluate(f.Arguments[0]); - if (monthValue is null) return null; - int month = Convert.ToInt32(monthValue, CultureInfo.InvariantCulture); - bool abbreviate = f.Arguments.Count > 1 && IsTrue(f.Arguments[1]); + if (Evaluate(f.Arguments[0]) is not { } month || Abbreviate(f) is not { } abbreviate) + return null; + int number = Setting(month, 1, 12); return abbreviate - ? EnUs.DateTimeFormat.GetAbbreviatedMonthName(month) - : EnUs.DateTimeFormat.GetMonthName(month); + ? EnUs.DateTimeFormat.GetAbbreviatedMonthName(number) + : EnUs.DateTimeFormat.GetMonthName(number); } + /// Access WeekdayName(weekday, [abbreviate], [firstdayofweek]): the English name of the day at + /// 1-based position weekday in a week starting on firstdayofweek, which defaults to the system's + /// first day (verified vs ACE: WeekdayName(1) is Monday under en-AU). A weekday outside 1-7 is an invalid + /// procedure call. private object? WeekdayNameOf(FunctionCall f) { - object? wdV = Evaluate(f.Arguments[0]); - if (wdV is null) return null; - int weekday = Convert.ToInt32(wdV, CultureInfo.InvariantCulture); - bool abbreviate = f.Arguments.Count > 1 && IsTrue(f.Arguments[1]); - int firstDay = f.Arguments.Count > 2 ? Convert.ToInt32(Evaluate(f.Arguments[2]), CultureInfo.InvariantCulture) : 1; - if (firstDay == 0) firstDay = 1; // vbUseSystem → treat as vbSunday for determinism - // Map to a 0..6 index into Sunday..Saturday. - int index = ((firstDay - 1) + (weekday - 1)) % 7; - if (index < 0) index += 7; - var names = abbreviate ? EnUs.DateTimeFormat.AbbreviatedDayNames : EnUs.DateTimeFormat.DayNames; - return names[index]; + if (Evaluate(f.Arguments[0]) is not { } weekday || Abbreviate(f) is not { } abbreviate + || FirstDayOfWeek(f, 2, absent: 0) is not { } first) + return null; + int index = ((int)first + Setting(weekday, 1, 7) - 1) % 7; + return (abbreviate ? EnUs.DateTimeFormat.AbbreviatedDayNames : EnUs.DateTimeFormat.DayNames)[index]; } + /// MonthName's and WeekdayName's abbreviate argument, read as CBool reads it; false when absent. + private bool? Abbreviate(FunctionCall f) => Optional(f, 1, false, VbaBool); + /// Access Partition(number, start, stop, interval): a "lower:upper" range label, both /// sides right-justified to a fixed width. Below the range → lower blank, upper = start-1; above → - /// lower = stop+1, upper blank; otherwise the interval bucket (verified vs ACE). NULL-propagating. + /// lower = stop+1, upper blank; otherwise the interval bucket (verified vs ACE). The arguments are read as + /// the conversion functions read them (a date as its serial); a negative start, a stop not after the start or an + /// interval below 1 is an invalid procedure call. NULL-propagating. private object? PartitionOf(FunctionCall f) { - object? nV = Evaluate(f.Arguments[0]); - if (nV is null) return null; - long number = Convert.ToInt64(nV, CultureInfo.InvariantCulture); - long start = Convert.ToInt64(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture); - long stop = Convert.ToInt64(Evaluate(f.Arguments[2]), CultureInfo.InvariantCulture); - long interval = Convert.ToInt64(Evaluate(f.Arguments[3]), CultureInfo.InvariantCulture); + object?[] args = f.Arguments.Select(Evaluate).ToArray(); + if (args.Any(a => a is null)) return null; + long number = Lng(ConversionNumber(args[0]!)); + long start = Lng(ConversionNumber(args[1]!)); + long stop = Lng(ConversionNumber(args[2]!)); + long interval = Lng(ConversionNumber(args[3]!)); + if (start < 0 || stop <= start || interval < 1) + throw new ArgumentException("Invalid procedure call: Partition needs 0 <= start < stop and an interval of at least 1."); // Fixed field width = the widest boundary that can appear (the below/above sentinels). int width = Math.Max((start - 1).ToString(CultureInfo.InvariantCulture).Length, @@ -825,35 +1025,32 @@ private static string FromUnicodeBytes(string s) private object? ByteLeft(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]); - if (sv is null) return null; + if (sv is null || CountArgument(f, 1) is not { } count) return null; byte[] b = ToBytes(sv); - int n = Math.Clamp(Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture), 0, b.Length); - return ByteResult(sv, b[..n]); + return ByteResult(sv, b[..Math.Min(count, b.Length)]); } /// VBA RightB(string, bytes): the trailing bytes bytes. NULL-propagating. private object? ByteRight(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]); - if (sv is null) return null; + if (sv is null || CountArgument(f, 1) is not { } count) return null; byte[] b = ToBytes(sv); - int n = Math.Clamp(Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture), 0, b.Length); - return ByteResult(sv, b[^n..]); + return ByteResult(sv, b[^Math.Min(count, b.Length)..]); } /// VBA MidB(string, startByte[, lenBytes]): a 1-based **byte** slice (may start/end - /// mid-character). NULL-propagating. + /// mid-character). NULL-propagating; a start below 1 or a negative length is an invalid procedure call. private object? ByteMid(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]); - if (sv is null) return null; + if (sv is null || CountArgument(f, 1, least: 1) is not { } first + || CountArgument(f, 2, absent: int.MaxValue) is not { } length) + return null; byte[] b = ToBytes(sv); - int start = Math.Max(0, Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture) - 1); + int start = first - 1; if (start >= b.Length) return ByteResult(sv, []); - int len = f.Arguments.Count > 2 - ? Convert.ToInt32(Evaluate(f.Arguments[2]), CultureInfo.InvariantCulture) - : b.Length - start; - len = Math.Clamp(len, 0, b.Length - start); + int len = Math.Min(length, b.Length - start); return ByteResult(sv, b[start..(start + len)]); } @@ -869,9 +1066,9 @@ private static string FromUnicodeBytes(string s) int argc = f.Arguments.Count; object? s1v = Evaluate(f.Arguments[argc >= 3 ? 1 : 0]); object? s2v = Evaluate(f.Arguments[argc >= 3 ? 2 : 1]); - if (s1v is null || s2v is null) return null; - int start = argc >= 3 ? Math.Max(0, Convert.ToInt32(Evaluate(f.Arguments[0]), CultureInfo.InvariantCulture) - 1) : 0; - int idx = IndexOfBytes(ToBytes(s1v), ToBytes(s2v), start); + if (s1v is null || s2v is null || (argc >= 3 ? CountArgument(f, 0, least: 1) : 1) is not { } start) + return null; + int idx = IndexOfBytes(ToBytes(s1v), ToBytes(s2v), start - 1); return idx < 0 ? 0 : idx + 1; } @@ -893,139 +1090,249 @@ private static int IndexOfBytes(byte[] hay, byte[] needle, int start) /// column as a UTF-16 string — LenB of a 3-byte value is 4). private static byte[] ToBytes(object v) { - if (v is not byte[] bytes) return System.Text.Encoding.Unicode.GetBytes(v.ToString()!); + if (v is not byte[] bytes) return Encoding.Unicode.GetBytes(ConcatText(v)); if (bytes.Length % 2 == 0) return bytes; var padded = new byte[bytes.Length + 1]; Array.Copy(bytes, padded, bytes.Length); return padded; } - /// Decodes a byte slice back to a string, dropping a trailing incomplete (odd) byte — matching ACE - /// (MidB(x, 1, 3) yields one character from three bytes). - private static string FromBytes(byte[] b) => - System.Text.Encoding.Unicode.GetString(b, 0, b.Length - (b.Length % 2)); - - /// The text a string function operates on: a binary value reinterpreted as a UTF-16LE string (odd - /// byte zero-padded), or a normal string. - private static string ToText(object v) => - v is byte[] ? System.Text.Encoding.Unicode.GetString(ToBytes(v)) : v.ToString()!; - - // --- Financial / formatting / colour functions (JES surface) --- - - private double FinArg(FunctionCall f, int i) => Convert.ToDouble(Evaluate(f.Arguments[i]), CultureInfo.InvariantCulture); - private double FinArgOr(FunctionCall f, int i, double def) - => f.Arguments.Count > i && Evaluate(f.Arguments[i]) is { } v ? Convert.ToDouble(v, CultureInfo.InvariantCulture) : def; - /// Optional decimal-count arg for the FormatX functions — default 2 (also for the VBA "-1" default). - private int FmtDigits(FunctionCall f, int i) - => f.Arguments.Count > i && Evaluate(f.Arguments[i]) is { } v && Convert.ToInt32(v, CultureInfo.InvariantCulture) >= 0 - ? Convert.ToInt32(v, CultureInfo.InvariantCulture) : 2; + /// + /// SQL Server's DATALENGTH, a LibRed extension (Access has none): the bytes a value takes as Access stores + /// it. Text is two per character — Access text is UTF-16, as nvarchar is — with trailing spaces counted and + /// no account of the on-disk Unicode compression; a binary value is its length, unpadded (LenB pads an odd one). + /// A Byte is 1, an Integer 2, a Long and a Single 4, a Double, a Currency, a date and a BIGINT 8, a GUID 16 and a + /// Decimal 17 — its sign byte and 16-byte magnitude. A Boolean is stored as a bit of the row's null bitmap and so + /// takes no byte of its own; it counts 1, as SQL Server counts a bit. + /// + private static int DataLength(object value, bool currency) => value switch + { + string s => checked(s.Length * 2), + char => 2, + byte[] bytes => bytes.Length, + bool or byte or sbyte => 1, + short or ushort => 2, + int or uint or float => 4, + long or ulong or double or DateTime => 8, + decimal => currency ? 8 : 17, + Guid => 16, + _ => checked(ConcatText(value).Length * 2), + }; - private object FormatDateTimeFn(FunctionCall f) + /// Successive byte pairs as UTF-16 code units (low byte first), dropping a trailing odd byte — matching + /// ACE (MidB(x, 1, 3) yields one character from three bytes). A lone surrogate is kept as it is. + private static string FromBytes(byte[] bytes) { - var dt = (DateTime)ToDate(Evaluate(f.Arguments[0])!); - int mode = f.Arguments.Count > 1 ? Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture) : 0; - CultureInfo c = CultureInfo.CurrentCulture; - return mode switch - { - 1 => dt.ToString("D", c), // Long Date - 2 => dt.ToString("d", c), // Short Date - 3 => dt.ToString("T", c), // Long Time - 4 => dt.ToString("t", c), // Short Time - _ => dt.TimeOfDay == TimeSpan.Zero ? dt.ToString("d", c) : dt.ToString("g", c), // General Date - }; + var chars = new char[bytes.Length / 2]; + for (int i = 0; i < chars.Length; i++) + chars[i] = (char)(bytes[2 * i] | (bytes[2 * i + 1] << 8)); + return new string(chars); } + // --- Colour functions --- + // QBColor maps 0..15 to fixed BGR Long values (verified vs ACE: QBColor(4) = 128). private static readonly int[] QbColors = [ 0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, 0x008080, 0xC0C0C0, 0x808080, 0xFF0000, 0x00FF00, 0xFFFF00, 0x0000FF, 0xFF00FF, 0x00FFFF, 0xFFFFFF, ]; - private static int QbColor(int n) => QbColors[((n % 16) + 16) % 16]; - - private static double Pow1(double rate, double nper) => Math.Pow(1 + rate, nper); - private static double AnnuityFactor(double rate, double nper, double type) => (1 + rate * type) * (Pow1(rate, nper) - 1) / rate; - /// VBA Pmt(rate, nper, pv, [fv=0], [type=0]): the constant payment for an annuity. - private object Pmt(FunctionCall f) + /// + /// Access RGB(red, green, blue): the colour as a Long, red in the low byte. NULL-propagating. Each part is a + /// setting () where above 255 is taken as 255 (verified vs ACE: RGB(256, 0, 0) is 255, and + /// RGB(-0.6, 0, 0) fails). + /// + private object? Rgb(FunctionCall f) { - double rate = FinArg(f, 0), nper = FinArg(f, 1), pv = FinArg(f, 2), fv = FinArgOr(f, 3, 0), type = FinArgOr(f, 4, 0); - return rate == 0 ? -(pv + fv) / nper : -(pv * Pow1(rate, nper) + fv) / AnnuityFactor(rate, nper, type); + if (Evaluate(f.Arguments[0]) is not { } red || Evaluate(f.Arguments[1]) is not { } green + || Evaluate(f.Arguments[2]) is not { } blue) + return null; + return Part(red) | Part(green) << 8 | Part(blue) << 16; + + static int Part(object v) => Math.Min(Setting(v, 0, short.MaxValue), 255); } - /// VBA FV(rate, nper, pmt, [pv=0], [type=0]): the future value of an annuity. - private object Fv(FunctionCall f) + // --- Financial functions --- + + /// + /// A financial function (verified vs ACE, to the last bit). The arguments are read as the conversion functions read + /// them (text as a number, a date as its serial, True as -1); an omitted one is 0, except Rate's guess and DDB's + /// factor. A "type" argument means payment at the start of each period whenever it is not 0. Arguments out of range + /// are an invalid procedure call; a result past a Double comes back as infinity or NaN, as ACE returns it. A Null + /// argument gives Null where ACE raises an error. + /// The algorithms and the order of their arithmetic are the VBA runtime's, as Microsoft.VisualBasic's + /// Financial module carries them; the order decides the last digit. + /// + private object? Financial(FunctionCall f, Func compute) { - double rate = FinArg(f, 0), nper = FinArg(f, 1), pmt = FinArg(f, 2), pv = FinArgOr(f, 3, 0), type = FinArgOr(f, 4, 0); - return rate == 0 ? -(pv + pmt * nper) : -(pv * Pow1(rate, nper) + pmt * AnnuityFactor(rate, nper, type)); + var arguments = new double[6]; + for (int i = 0; i < f.Arguments.Count; i++) + { + if (Evaluate(f.Arguments[i]) is not { } value) + return null; + arguments[i] = Dbl(ConversionNumber(value)); + } + return compute(arguments); } - /// VBA PV(rate, nper, pmt, [fv=0], [type=0]): the present value of an annuity. - private object Pv(FunctionCall f) + /// A "type" argument: payment at the start of each period when it rounds (half to even) to anything but 0 + /// (verified vs ACE: 0.5 is the end, 1.5, 2 and -1 the start). + private static bool Due(double type) => Math.Round(type, MidpointRounding.ToEven) != 0; + + private static ArgumentException OutOfRange(string argument) => + new($"Invalid procedure call: {argument} is out of range."); + + /// Access Pmt(rate, nper, pv, [fv], [type]): the payment each period. An nper of 0 is out of range. + private static double Pmt(double rate, double nper, double pv, double fv, bool due) { - double rate = FinArg(f, 0), nper = FinArg(f, 1), pmt = FinArg(f, 2), fv = FinArgOr(f, 3, 0), type = FinArgOr(f, 4, 0); - return rate == 0 ? -(fv + pmt * nper) : -(fv + pmt * AnnuityFactor(rate, nper, type)) / Pow1(rate, nper); + if (nper == 0) + throw OutOfRange("NPer"); + if (rate == 0) + return (-fv - pv) / nper; + double start = due ? 1 + rate : 1; + double growth = Math.Pow(rate + 1, nper); + return (-fv - pv * growth) / (start * (growth - 1)) * rate; } - /// VBA NPer(rate, pmt, pv, [fv=0], [type=0]): the number of periods for an annuity. - private object NPer(FunctionCall f) + /// Access FV(rate, nper, pmt, [pv], [type]): the value after the last payment. + private static double Fv(double rate, double nper, double pmt, double pv, bool due) { - double rate = FinArg(f, 0), pmt = FinArg(f, 1), pv = FinArg(f, 2), fv = FinArgOr(f, 3, 0), type = FinArgOr(f, 4, 0); - if (rate == 0) return -(pv + fv) / pmt; - double a = pmt * (1 + rate * type); - return Math.Log((a - fv * rate) / (a + pv * rate)) / Math.Log(1 + rate); + if (rate == 0) + return -pv - pmt * nper; + double start = due ? 1 + rate : 1; + double growth = Math.Pow(1 + rate, nper); + return -(pv * growth + pmt * start * (growth - 1) / rate); } - /// VBA IPmt(rate, per, nper, pv, [fv=0], [type=0]): the interest portion of payment per. - private object IPmt(FunctionCall f) + /// Access PV(rate, nper, pmt, [fv], [type]): the value before the first payment. + private static double Pv(double rate, double nper, double pmt, double fv, bool due) { - double rate = FinArg(f, 0), per = FinArg(f, 1), nper = FinArg(f, 2), pv = FinArg(f, 3), fv = FinArgOr(f, 4, 0), type = FinArgOr(f, 5, 0); - double pmt = rate == 0 ? -(pv + fv) / nper : -(pv * Pow1(rate, nper) + fv) / AnnuityFactor(rate, nper, type); - if (type == 1 && per == 1) return 0.0; - double balance = pv * Pow1(rate, per - 1) + pmt * (rate == 0 ? per - 1 : AnnuityFactor(rate, per - 1, type)); - double interest = -balance * rate; // interest paid is a cash outflow (negative), like Pmt - return type == 1 ? interest / (1 + rate) : interest; + if (rate == 0) + return -fv - pmt * nper; + double start = due ? 1 + rate : 1; + double growth = Math.Pow(1 + rate, nper); + return -(fv + pmt * start * (growth - 1) / rate) / growth; + } + + /// Access NPer(rate, pmt, pv, [fv], [type]): the number of payments. A rate of -1 or less, no + /// payment at a rate of 0, or payments that never reach the future value are out of range. + private static double NPer(double rate, double pmt, double pv, double fv, bool due) + { + if (rate <= -1) + throw OutOfRange("Rate"); + if (rate == 0) + return pmt == 0 ? throw OutOfRange("Pmt") : -(pv + fv) / pmt; + double payment = due ? pmt * (1 + rate) / rate : pmt / rate; + double future = -fv + payment, present = pv + payment; + if (future < 0 && present < 0) + (future, present) = (-future, -present); + else if (future <= 0 || present <= 0) + throw OutOfRange("Pmt"); + return (Math.Log(future) - Math.Log(present)) / Math.Log(rate + 1); } - /// VBA PPmt(...): the principal portion of a payment (= Pmt − IPmt). - private object PPmt(FunctionCall f) + /// Access IPmt(rate, per, nper, pv, [fv], [type]): the interest in payment per, which must be + /// above 0 and below nper + 1. + private static double IPmt(double rate, double per, double nper, double pv, double fv, bool due) + { + if (per <= 0 || per >= nper + 1) + throw OutOfRange("Per"); + if (due && per == 1) + return 0; + double pmt = Pmt(rate, nper, pv, fv, due); + if (due) + pv += pmt; + return Fv(rate, per - (due ? 2 : 1), pmt, pv, false) * rate; + } + + /// Access PPmt(rate, per, nper, pv, [fv], [type]): the principal in payment per. + private static double PPmt(double rate, double per, double nper, double pv, double fv, bool due) { - double rate = FinArg(f, 0), nper = FinArg(f, 2), pv = FinArg(f, 3), fv = FinArgOr(f, 4, 0), type = FinArgOr(f, 5, 0); - double pmt = rate == 0 ? -(pv + fv) / nper : -(pv * Pow1(rate, nper) + fv) / AnnuityFactor(rate, nper, type); - return pmt - Convert.ToDouble(IPmt(f), CultureInfo.InvariantCulture); + if (per <= 0 || per >= nper + 1) + throw OutOfRange("Per"); + return Pmt(rate, nper, pv, fv, due) - IPmt(rate, per, nper, pv, fv, due); } - /// VBA DDB(cost, salvage, life, period, [factor=2]): double-declining-balance depreciation. - private object Ddb(FunctionCall f) + /// + /// Access Rate(nper, pmt, pv, [fv], [type], [guess]): the rate per period, found by the secant method from the + /// guess (0.1 when omitted). An nper of 0 or less, or no answer within 40 steps, is out of range. + /// + private static double Rate(double nper, double pmt, double pv, double fv, bool due, double guess) { - double cost = FinArg(f, 0), salvage = FinArg(f, 1), life = FinArg(f, 2), period = FinArg(f, 3), factor = FinArgOr(f, 4, 2); - double rate = factor / life; - double bookStart = cost * Math.Pow(1 - rate, period - 1); - double dep = bookStart * rate; - if (bookStart - dep < salvage) dep = Math.Max(0, bookStart - salvage); - return dep; + const double step = 0.00001, epsilon = 0.0000001; + if (nper <= 0) + throw OutOfRange("NPer"); + double Error(double rate) + { + if (rate == 0) + return pv + pmt * nper + fv; + double growth = Math.Pow(rate + 1, nper); + double start = due ? 1 + rate : 1; + return pv * growth + pmt * start * (growth - 1) / rate + fv; + } + + double rate0 = guess, error0 = Error(rate0); + double rate1 = error0 > 0 ? rate0 / 2 : rate0 * 2, error1 = Error(rate1); + for (int i = 0; i < 40; i++) + { + if (error1 == error0) + { + rate0 = rate1 > rate0 ? rate0 - step : rate0 + step; + error0 = Error(rate0); + if (error1 == error0) + throw OutOfRange("Rate"); + } + rate0 = rate1 - (rate1 - rate0) * error1 / (error1 - error0); + error0 = Error(rate0); + if (Math.Abs(error0) < epsilon) + return rate0; + (error0, error1) = (error1, error0); + (rate0, rate1) = (rate1, rate0); + } + throw OutOfRange("Rate"); } - /// VBA Rate(nper, pmt, pv, [fv=0], [type=0], [guess=0.1]): the per-period rate, solved by - /// Newton–Raphson on the annuity equation. - private object Rate(FunctionCall f) + /// Access SLN(cost, salvage, life): straight-line depreciation. A life of 0 is out of range. + private static double Sln(double cost, double salvage, double life) => + life == 0 ? throw OutOfRange("Life") : (cost - salvage) / life; + + /// Access SYD(cost, salvage, life, period): sum-of-years'-digits depreciation. A negative salvage, or a + /// period of 0 or less or past the life, is out of range. + private static double Syd(double cost, double salvage, double life, double period) { - double nper = FinArg(f, 0), pmt = FinArg(f, 1), pv = FinArg(f, 2), fv = FinArgOr(f, 3, 0), type = FinArgOr(f, 4, 0); - double r = FinArgOr(f, 5, 0.1); - for (int iter = 0; iter < 100; iter++) + if (salvage < 0 || period > life || period <= 0) + throw OutOfRange("Period"); + return (cost - salvage) * (life - period + 1) / (life * (life + 1) / 2); + } + + /// + /// Access DDB(cost, salvage, life, period, [factor]): declining-balance depreciation at factor (2 when + /// omitted) over the life. A factor of 0 or less, a negative salvage, or a period of 0 or less or past the life is + /// out of range. A cost of 0 or less depreciates nothing, and a period's depreciation never takes the value below + /// the salvage. + /// + private static double Ddb(double cost, double salvage, double life, double period, double factor) + { + if (factor <= 0 || salvage < 0 || period <= 0 || period > life) + throw OutOfRange("Period"); + if (cost <= 0) + return 0; + if (life < 2) + return cost - salvage; + if (life == 2) + return period > 1 ? 0 : cost - salvage; + if (period <= 1) { - double v = r == 0 ? pv + pmt * nper + fv - : pv * Pow1(r, nper) + pmt * (1 + r * type) * (Pow1(r, nper) - 1) / r + fv; - const double h = 1e-6; - double vh = (r + h) == 0 ? pv + pmt * nper + fv - : pv * Pow1(r + h, nper) + pmt * (1 + (r + h) * type) * (Pow1(r + h, nper) - 1) / (r + h) + fv; - double deriv = (vh - v) / h; - if (Math.Abs(deriv) < 1e-12) break; - double next = r - v / deriv; - if (Math.Abs(next - r) < 1e-10) return next; - r = next; + double first = cost * factor / life, most = cost - salvage; + return first > most ? most : first; } - return r; + double remaining = (life - factor) / life; + double depreciation = factor * cost / life * Math.Pow(remaining, period - 1); + double excess = cost * (1 - Math.Pow(remaining, period)) - cost + salvage; + if (excess > 0) + depreciation -= excess; + return depreciation >= 0 ? depreciation : 0; } private uint _localRandSeed = 0x50000; // used when no connection-scoped SessionState is available @@ -1035,11 +1342,13 @@ private object Rate(FunctionCall f) /// number = 0 repeats the last value; number < 0 reseeds deterministically from the /// argument's Single bit pattern. The seed is connection-scoped (via ) since the /// JES has no Randomize. Result is a Single widened to Double, matching ACE. - private object Rnd(FunctionCall f) + private object? Rnd(FunctionCall f) { uint seed = session?.RandSeed ?? _localRandSeed; - double arg = f.Arguments.Count > 0 && Evaluate(f.Arguments[0]) is { } a - ? Convert.ToDouble(a, CultureInfo.InvariantCulture) : 1.0; + // The argument is read as a Single, as conversion functions read it; Null gives Null (ACE raises + // "Invalid use of Null"). + if (Optional(f, 0, 1.0, a => Finite(Sng(ConversionNumber(a)))) is not { } arg) + return null; if (arg != 0) { if (arg < 0) @@ -1061,23 +1370,50 @@ private static int RandomLong() return value; } - /// Access ROUND(number[, digits]): banker's rounding, preserving the operand's type (a double - /// rounds to a double, a decimal to a decimal — the EF contract). NULL-propagating. + /// + /// Access Round(number[, places]) (verified vs ACE): half to even on the number's decimal form — a Double or + /// Single as OLE Automation writes it, so Round(2.675, 2) is 2.68 and Round('2.55', 1) 2.6 — and the + /// number unchanged when nothing is cut. The operand's type is kept (the EF contract); text and dates give a + /// Double and True is -1; a number written with a decimal point rounds as written. Places are read as a number and + /// rounded; Null places give Null, fewer than 0 are an invalid procedure call, and more than a Decimal holds leave + /// the number as it is. + /// private object? Round(FunctionCall f) { - object? value = Evaluate(f.Arguments[0]); - if (value is null) return null; - int digits = f.Arguments.Count > 1 - ? Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture) - : 0; + if (Evaluate(f.Arguments[0]) is not { } value || Optional(f, 1, 0, v => Count(v)) is not { } places) + return null; + places = Math.Min(places, 28); + return value switch { - decimal m => Math.Round(m, digits, MidpointRounding.ToEven), - double d => Math.Round(d, digits, MidpointRounding.ToEven), - float s => (float)Math.Round((double)s, digits, MidpointRounding.ToEven), - long l => (long)Math.Round((decimal)l, digits, MidpointRounding.ToEven), - _ => (int)Math.Round(Convert.ToDecimal(value, CultureInfo.InvariantCulture), digits, MidpointRounding.ToEven), + decimal m => decimal.Round(m, places, MidpointRounding.ToEven), + double d when WrittenValue(f.Arguments[0]) is decimal written => + RoundWritten(d, places, () => written, r => (double)r), + double d => RoundDouble(d, places), + float s => RoundWritten(s, places, () => JetDecimalConverter.FromSingle(s), r => (float)r), + long or ulong => Lng(value), + int or short or byte or bool => Int(value), + _ => RoundDouble(Dbl(Serial(NumericOperand(value)!)), places), }; + + static object RoundDouble(double d, int places) => + RoundWritten(d, places, () => JetDecimalConverter.FromDouble(d), r => (double)r); + + // A number past a Decimal has no places left to cut. + static object RoundWritten(T original, int places, Func written, Func back) where T : notnull + { + decimal exact; + try + { + exact = written(); + } + catch (OverflowException) + { + return original; + } + decimal rounded = decimal.Round(exact, places, MidpointRounding.ToEven); + return rounded == exact ? original : back(rounded); + } } /// Applies a conversion to a single argument, propagating NULL. @@ -1087,29 +1423,62 @@ private static int RandomLong() return value is null ? null : convert(value); } - /// Access CDate: a date passes through, a string is parsed, a number is an OLE Automation date - /// (days since 1899-12-30). - private static object ToDate(object v) => v switch + /// + /// Access CDate (verified vs ACE): a date passes through; text is read as a date and time the way OLE + /// Automation reads it (), and otherwise as a number; a number is the date at that + /// serial (True is -1, 1899-12-29). + /// + private static DateTime ToDate(object v) => v switch { DateTime d => d, - string s => DateTime.Parse(s, CultureInfo.InvariantCulture), - _ => DateTime.FromOADate(Convert.ToDouble(v, CultureInfo.InvariantCulture)), + _ when TryDateText(v, out DateTime parsed) => parsed, + _ => OaDate(Dbl(ConversionNumber(v))), }; - /// Access IsDate: true only for a date value or a string that parses as a date/time. A number, - /// NULL, or an unrecognisable string is false (verified vs ACE — unlike CDate, a bare number is not a - /// date here). - private static bool IsDateValue(object? v) => v switch + /// A conversion function's argument as a number: text read as one (), a date + /// as its serial and a Boolean as -1 or 0; a GUID or binary value is a type mismatch. + internal static object ConversionNumber(object v) => Numeric(Serial(NumericOperand(v)!)); + + /// + /// A CCur or CDec argument as a Decimal, exactly where it can be: a number written with a decimal + /// point as written, and text that reads as a number within a Decimal's range as it reads (verified vs ACE: + /// CCur('12345678901234.5678') and CCur(12345678901234.5678) keep every place). Anything else is read as the + /// conversion functions read it. Null for a Null argument. + /// + private object? DecimalArgument(FunctionCall f, Func convert) { - null => false, - DateTime => true, - string s => DateTime.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out _), - _ => false, - }; + if (WrittenValue(f.Arguments[0]) is decimal written) + return convert(written); + return Evaluate(f.Arguments[0]) switch + { + null => null, + (string or char) and var text when TextAsDecimal(text.ToString()!) is decimal exact => convert(exact), + var value => convert(ArithmeticDecimal(ConversionNumber(value))), + }; + } - /// A (string, int) → string function (LEFT/RIGHT), propagating NULL on the string argument. - // Left/Right: NULL string propagates; a NULL length raises "Data type mismatch" and a negative length - // "Invalid procedure call" — matching ACE (which errors rather than clamping). A zero length yields "". + /// Access CCur: the value to four places, half to even; past a Currency is an overflow. + private static decimal ToCurrency(decimal number) + { + decimal value = decimal.Round(number, 4, MidpointRounding.ToEven); + return Math.Abs(value) <= 922337203685477.5807m || value == -922337203685477.5808m + ? value + : throw new OverflowException($"Overflow: {value} is outside the range of a Currency."); + } + + /// A date, or text that reads as one (); a number or Null is not. + private static bool TryDateText(object? v, out DateTime value) + { + value = v as DateTime? ?? default; + return v is DateTime || v is string text && VbaDateText.TryParse(text, CultureInfo.CurrentCulture, out value); + } + + /// A DateValue or TimeValue argument: a date or text that reads as one, else a type mismatch. + private static DateTime DateValueArgument(object v) => + TryDateText(v, out DateTime value) ? value : throw new InvalidCastException($"Type mismatch: '{ConcatText(v)}' is not a date."); + + /// Left and Right: the text's first or last characters. A Null length raises "Data type mismatch" as + /// ACE does, and a negative one is an invalid procedure call. private object? StringInt(FunctionCall f, Func op) { object? s = Evaluate(f.Arguments[0]); @@ -1117,59 +1486,48 @@ private static int RandomLong() object? nv = Evaluate(f.Arguments[1]); if (nv is null) throw new InvalidOperationException("Data type mismatch in criteria expression: length argument is null."); - int n = Convert.ToInt32(nv, CultureInfo.InvariantCulture); - if (n < 0) - throw new InvalidOperationException("Invalid procedure call: length cannot be negative."); - return op(s.ToString()!, n); + return op(ConcatText(s), Count(nv)); } - /// Access MID(string, start[, length]) — a 1-based substring; length omitted means to the end. + /// Access MID(string, start[, length]) — a 1-based substring; length omitted means to the end. A start + /// below 1 or a negative length is an invalid procedure call. private object? Mid(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]); - if (sv is null) return null; // Mid propagates NULL on the string argument - string s = sv.ToString()!; - int start = Convert.ToInt32(Evaluate(f.Arguments[1]), CultureInfo.InvariantCulture); - if (start < 1) // ACE errors rather than clamping - throw new InvalidOperationException("Invalid procedure call: Mid() start must be >= 1."); + if (sv is null || CountArgument(f, 1, least: 1) is not { } start + || CountArgument(f, 2, absent: int.MaxValue) is not { } length) + return null; + string s = ConcatText(sv); int from = start - 1; if (from >= s.Length) return ""; - int avail = s.Length - from; - int len = avail; - if (f.Arguments.Count > 2 && Evaluate(f.Arguments[2]) is { } lenVal) - { - int requested = Convert.ToInt32(lenVal, CultureInfo.InvariantCulture); - if (requested < 0) // ACE errors on a negative length - throw new InvalidOperationException("Invalid procedure call: Mid() length cannot be negative."); - len = Math.Min(requested, avail); - } - return s.Substring(from, len); + return s.Substring(from, Math.Min(length, s.Length - from)); } - /// Access INSTR([start,] string1, string2[, compare]) — the 1-based position of string2 in - /// string1 (0 if not found). start defaults to 1; compare 0 = binary (case-sensitive), else text. + /// Access INSTR([start,] string1, string2[, compare]) — the 1-based position of string2 in string1, 0 if + /// it is not there. An empty string2 is found at start, wherever that is, unless string1 is empty; a start below 1 + /// is an invalid procedure call. private object? Instr(FunctionCall f) { int argc = f.Arguments.Count; // 2 args: (s1, s2); 3+: (start, s1, s2[, compare]). - int start = argc >= 3 ? Convert.ToInt32(Evaluate(f.Arguments[0]), CultureInfo.InvariantCulture) : 1; + if ((argc >= 3 ? CountArgument(f, 0, least: 1) : 1) is not { } start) + return null; object? s1v = Evaluate(f.Arguments[argc >= 3 ? 1 : 0]); object? s2v = Evaluate(f.Arguments[argc >= 3 ? 2 : 1]); - if (s1v is null || s2v is null) return null; - StringComparison cmp = argc >= 4 && Convert.ToInt32(Evaluate(f.Arguments[3]), CultureInfo.InvariantCulture) == 0 - ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + if (s1v is null || s2v is null || BinaryCompare(f, 3) is not { } binary) + return null; // A byte[] argument (e.g. a 0xNN hex literal needle, or a binary haystack) coerces to a string one // char per byte, matching Jet — not "System.Byte[]". string s1 = ToJetString(s1v), s2 = ToJetString(s2v); - if (start < 1) start = 1; + if (s1.Length == 0) return 0; + if (s2.Length == 0) return start; if (start > s1.Length) return 0; - int idx = s1.IndexOf(s2, start - 1, cmp); - return idx < 0 ? 0 : idx + 1; + return FindText(s1, s2, start - 1, binary).Index + 1; } - /// Access REPLACE(string, find, replace[, start[, count[, compare]]]) — replaces occurrences of - /// find (from the 1-based start, at most count times, case-insensitive by default). + /// Access REPLACE(string, find, replace[, start[, count[, compare]]]) — the text from start on, with + /// find replaced at most count times (all when count is -1). A start below 1 is an invalid procedure call. private object? Replace(FunctionCall f) { object? sv = Evaluate(f.Arguments[0]), findv = Evaluate(f.Arguments[1]), replv = Evaluate(f.Arguments[2]); @@ -1179,14 +1537,12 @@ private static int RandomLong() // changes, only ones that error start returning NULL. if (sv is null || findv is null || replv is null) return null; - string s = sv.ToString()!, find = findv.ToString()!, repl = replv.ToString()!; + string s = ConcatText(sv), find = ConcatText(findv), repl = ConcatText(replv); - int start = f.Arguments.Count > 3 ? Convert.ToInt32(Evaluate(f.Arguments[3]), CultureInfo.InvariantCulture) : 1; - if (start < 1) // ACE errors rather than clamping - throw new InvalidOperationException("Invalid procedure call: Replace() start must be >= 1."); - int count = f.Arguments.Count > 4 ? Convert.ToInt32(Evaluate(f.Arguments[4]), CultureInfo.InvariantCulture) : -1; - StringComparison cmp = f.Arguments.Count > 5 && Convert.ToInt32(Evaluate(f.Arguments[5]), CultureInfo.InvariantCulture) == 0 - ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + if (CountArgument(f, 3, least: 1, absent: 1) is not { } start + || CountArgument(f, 4, least: -1, absent: -1) is not { } count + || BinaryCompare(f, 5) is not { } binary) + return null; if (start > s.Length) return ""; s = s[(start - 1)..]; @@ -1196,114 +1552,321 @@ private static int RandomLong() int pos = 0, replaced = 0; while (true) { - int j = (count >= 0 && replaced >= count) ? -1 : s.IndexOf(find, pos, cmp); + (int j, int length) = count >= 0 && replaced >= count ? (-1, 0) : FindText(s, find, pos, binary); if (j < 0) { sb.Append(s.AsSpan(pos)); break; } sb.Append(s, pos, j - pos).Append(repl); - pos = j + find.Length; + pos = j + length; replaced++; } return sb.ToString(); } - /// Applies a numeric transform to a single argument, propagating NULL. - /// A numeric transform (Fix/Int/Abs) that **preserves the operand's type** (double→double, - /// single→single, decimal→decimal, int→int, long→long) so it matches EF's Math.* return type. Integer - /// types use the exact decimal op (no floating round-trip). NULL-propagating. - private object? Numeric1(FunctionCall f, Func dOp, Func mOp) => + /// + /// Fix, Int and Abs, keeping the operand's type (double→double, single→single, decimal→decimal, long→long, and + /// the narrower integers and Booleans→int) so it matches EF's Math.* return type; NULL-propagating. Text is read + /// as a number and gives a Double, True is -1, and an Integer result past a Long widens to a Double (verified vs + /// ACE: Abs(-2147483648) is 2147483648). A date is its serial: Int and Fix give a date back + /// (), Abs a Double. + /// + private object? Numeric1(FunctionCall f, Func dOp, Func mOp, bool keepsDate) => Evaluate(f.Arguments[0]) switch { null => null, decimal m => mOp(m), double d => dOp(d), float s => (float)dOp(s), - long l => (long)mOp(l), - var v => (int)mOp(Convert.ToDecimal(v!, CultureInfo.InvariantCulture)), + long l => checked((long)mOp(l)), + DateTime d => keepsDate ? OaDate(dOp(d.ToOADate())) : dOp(d.ToOADate()), + var v when v is string or char or Guid or byte[] => dOp(Dbl(NumericOperand(v)!)), + var v => mOp(Dec(v)) is var r && r >= int.MinValue && r <= int.MaxValue ? (object)(int)r : (double)r, }; - /// Applies a double-precision transform to a single argument, propagating NULL. Used for - /// the trig/exp/log/sqrt VBA functions, which are inherently floating-point. + /// + /// Sqr, Exp, Log, Sin, Cos, Tan and Atn, on a Double read as the conversion functions read it; NULL-propagating. + /// A result that is not a number — the square root or log of a number below its domain, or a trigonometric + /// function of a number too large to reduce — is an invalid procedure call, and a result past a Double an + /// overflow (verified vs ACE). + /// private object? UnaryDouble(FunctionCall f, Func op) { - object? value = Evaluate(f.Arguments[0]); - return value is null ? null : op(Convert.ToDouble(value, CultureInfo.InvariantCulture)); + if (Evaluate(f.Arguments[0]) is not { } value) + return null; + double x = Dbl(ConversionNumber(value)); + double result = op(x); + return double.IsNaN(result) || double.IsNegativeInfinity(result) && x == 0 + ? throw new ArgumentException($"Invalid procedure call: the function is not defined at {x}.") + : Finite(result); } - /// Access DATEPART(interval, date): extracts a component of a date as an int. - private static object? DatePart(object? interval, object? date) + /// + /// A function of two Doubles, read as reads one; NULL-propagating. A result that is not + /// a number is an invalid procedure call, and a result past a Double an overflow. + /// + private object? BinaryDouble(FunctionCall f, Func op) { - if (date is null) return null; - var d = Convert.ToDateTime(date, CultureInfo.InvariantCulture); - return (interval?.ToString() ?? "").ToLowerInvariant() switch + if (Evaluate(f.Arguments[0]) is not { } left || Evaluate(f.Arguments[1]) is not { } right) + return null; + double x = Dbl(ConversionNumber(left)), y = Dbl(ConversionNumber(right)); + double result = op(x, y); + return double.IsNaN(result) + ? throw new ArgumentException($"Invalid procedure call: {f.Name} is not defined at {x}, {y}.") + : Finite(result); + } + + /// A trigonometric function, which ACE refuses for an argument of 9.223372E18 or more (verified). + private static Func Trigonometric(Func op) => + x => Math.Abs(x) >= 9.223372E18 ? double.NaN : op(x); + + /// Access DatePart(interval, date, [firstdayofweek], [firstweekofyear]): a component of a date. + /// "ms", "mcs" and "ns" are LibRed extensions. + private object? DatePart(FunctionCall f) + { + if (Evaluate(f.Arguments[0]) is not { } interval || Evaluate(f.Arguments[1]) is not { } date + || FirstDayOfWeek(f, 2) is not { } first || FirstWeekOfYear(f, 3) is not { } rule) + return null; + DateTime d = ToDate(date); + return ConcatText(interval).ToLowerInvariant() switch { "yyyy" => d.Year, "q" => (d.Month + 2) / 3, "m" => d.Month, "y" => d.DayOfYear, "d" => d.Day, - "w" => (int)d.DayOfWeek + 1, - "ww" => CultureInfo.InvariantCulture.Calendar.GetWeekOfYear(d, CalendarWeekRule.FirstDay, DayOfWeek.Sunday), + "w" => DaysIntoWeek(d, first) + 1, + "ww" => WeekOfYear(d, first, rule), "h" => d.Hour, "n" => d.Minute, "s" => d.Second, "ms" => d.Millisecond, "mcs" => d.Microsecond, "ns" => d.Nanosecond, - _ => throw new NotSupportedException($"DATEPART interval '{interval}' is not supported."), + _ => throw UnknownInterval(interval), }; } - /// A bitwise op (Access BAND/BOR/BXOR) over integer operands; the result - /// keeps the operand's int type (Int32, or Int64 if either operand is long). - private static object BitwiseOp(object a, object b, Func op) => - a is long or ulong || b is long or ulong ? (object)op(Lng(a), Lng(b)) : (int)op(Int(a), Int(b)); + private static ArgumentException UnknownInterval(object interval) => + new($"Invalid procedure call: '{interval}' is not a date interval."); + + /// Access Weekday(date, [firstdayofweek]): the day's 1-based position in the week. + private object? Weekday(FunctionCall f) => + Evaluate(f.Arguments[0]) is { } date && FirstDayOfWeek(f, 1) is { } first + ? DaysIntoWeek(ToDate(date), first) + 1 + : null; + + /// + /// A firstdayofweek argument: 1 (Sunday) to 7 (Saturday), or 0 for the system's first day; anything else is an + /// invalid procedure call (verified vs ACE: 0 is Monday under en-AU, and 8 and -1 are refused). Null when the + /// argument is Null. + /// + private DayOfWeek? FirstDayOfWeek(FunctionCall f, int index, int absent = 1) => + Optional(f, index, absent, v => Setting(v, 0, 7)) switch + { + null => null, + 0 => CultureInfo.CurrentCulture.DateTimeFormat.FirstDayOfWeek, + var day => (DayOfWeek)(day - 1), + }; + + /// A firstweekofyear argument: 1 (the week of January 1), 2 (the first week with four days), 3 (the first + /// full week), or 0 for the system's rule; anything else is an invalid procedure call. Null when it is Null. + private CalendarWeekRule? FirstWeekOfYear(FunctionCall f, int index) => + Optional(f, index, 1, v => Setting(v, 0, 3)) switch + { + null => null, + 0 => CultureInfo.CurrentCulture.DateTimeFormat.CalendarWeekRule, + 1 => CalendarWeekRule.FirstDay, + 2 => CalendarWeekRule.FirstFourDayWeek, + _ => CalendarWeekRule.FirstFullWeek, + }; + + /// A setting read as CInt reads it (half to even; verified vs ACE: '2' and 1.5 are 2); outside + /// - is an invalid procedure call. + private static int Setting(object v, int lowest, int highest) => InRange(AsInteger(v), lowest, highest); - /// A function of a single date argument (Year/Month/Day/…), NULL-propagating. - private object? DatePartOf(FunctionCall f, Func part) + /// How many days of the week come before the date's day, for a week starting on . + private static int DaysIntoWeek(DateTime date, DayOfWeek first) => ((int)date.DayOfWeek - (int)first + 7) % 7; + + /// + /// The date's week number (verified vs ACE). Days before the year's first week belong to the last week of the year + /// before, and under the four-day rule a week with four days in the next year is that year's week 1. + /// + private static int WeekOfYear(DateTime date, DayOfWeek first, CalendarWeekRule rule) { - object? v = Evaluate(f.Arguments[0]); - return v is null ? null : part(Convert.ToDateTime(v, CultureInfo.InvariantCulture)); + date = date.Date; + if (date < WeekOne(date.Year)) + return (date - WeekOne(date.Year - 1)).Days / 7 + 1; + if (rule == CalendarWeekRule.FirstFourDayWeek && date.Year < 9999 && date >= WeekOne(date.Year + 1)) + return 1; + return (date - WeekOne(date.Year)).Days / 7 + 1; + + DateTime WeekOne(int year) + { + var january1 = new DateTime(year, 1, 1); + int before = DaysIntoWeek(january1, first); + return january1.AddDays(rule switch + { + CalendarWeekRule.FirstDay => -before, + CalendarWeekRule.FirstFourDayWeek => before <= 3 ? -before : 7 - before, + _ => before == 0 ? 0 : 7 - before, + }); + } + } + + /// + /// BAND, BOR and BXOR on the operands' bits. When either operand is 16 bits, only the low 16 + /// bits are combined and the rest are the left operand's: its own upper bits, or the sign of the 16-bit result when + /// it is 16 bits itself (verified vs ACE: 1 BOR TRUE is 65535, CLNG(-1) BAND CINT(1) is -65535, CINT(1) BOR 70000 + /// is 4465, CINT(-2) BOR 1 is -1). Otherwise the result has the wider operand's width. The result is an Integer + /// only when both operands are 16 bits, and otherwise a Long or an Int64 (verified vs ACE: TRUE BAND CINT(-2) is + /// an Integer, CINT(-2) BAND 70000 and a Byte BAND a Byte are Longs). + /// + private static object BitwiseOp(object a, object b, Func op) + { + (long left, int leftWidth) = BitOperand(a); + (long right, int rightWidth) = BitOperand(b); + if (leftWidth != 16 && rightWidth != 16) + return Signed(op(left, right), Math.Max(leftWidth, rightWidth)); + + long low = op(left, right) & 0xFFFF; + return leftWidth != 16 ? Signed(left & ~0xFFFFL | low, leftWidth) + : rightWidth == 16 ? Signed(low, 16) + : Signed((short)low, 32); + } + + /// + /// Unary minus, keeping the operand's type as C# does (an Integer, a Byte or a Boolean gives a Long). Text is read as + /// a number first, and a date negates its serial and stays a date (verified vs ACE: -#2020-01-02# is 1779-12-27). A + /// result the operand's type cannot hold is an overflow (verified vs ACE: -CINT(-32768) and -CLNG(-2147483648)). + /// + private static object Negate(object v) => NumericOperand(v)! switch + { + decimal d => -d, + double d => -d, + float f => -f, + long l => checked(-l), + ulong u => checked(-(long)u), + DateTime d => OaDate(-d.ToOADate()), + short.MinValue => throw new OverflowException("Overflow: an Integer cannot hold 32768."), + var n => checked(-Int(n)), + }; + + /// BNOT: every bit of the operand flipped, an Integer for a Boolean or an Integer (verified vs ACE: + /// BNOT of a Yes/No column is an Integer, of a Byte a Long). + private static object BitNot(object v) + { + (long bits, int width) = BitOperand(v); + return Signed(~bits, width); } - /// DateSerial(y,m,d) / TimeSerial(h,m,s): build a date/time from three integer parts (parts may - /// be out of range and roll over, matching Access). NULL-propagating. + /// + /// An operand's bits, unsigned, and how many there are: 16 for a Boolean (True is all 16 set) or an Integer, 32 for a + /// Byte or a Long, 64 for an Int64. Anything else is first read as a whole number, as \ reads it: text as a + /// number, a date as its serial, then rounded half to even, and it must fit a Long. + /// + private static (long Bits, int Width) BitOperand(object v) => v switch + { + bool b => (b ? 0xFFFF : 0, 16), + short s => ((ushort)s, 16), + byte b => (b, 32), + int i => ((uint)i, 32), + long l => (l, 64), + ulong u => ((long)u, 64), + _ => ((uint)Int(Serial(NumericOperand(v)!)), 32), + }; + + /// The low bits read as a signed number: an Int16 for 16 bits, an Int32 for 32, + /// an Int64 for 64. + private static object Signed(long bits, int width) => width switch + { + // Each arm boxed on its own, or the switch would widen them all to Int64. + 16 => (object)(short)bits, + 32 => (object)(int)bits, + _ => (object)bits, + }; + + /// DateSerial and TimeSerial: a date or time from three Integer parts, read as CInt reads them (verified vs + /// ACE: 32768 is an overflow). Parts out of their range carry into the next. private object? DateParts(FunctionCall f, Func build) { - object? a = Evaluate(f.Arguments[0]), b = Evaluate(f.Arguments[1]), c = Evaluate(f.Arguments[2]); - if (a is null || b is null || c is null) return null; - return build(Int(a), Int(b), Int(c)); + if (Evaluate(f.Arguments[0]) is not { } a || Evaluate(f.Arguments[1]) is not { } b || Evaluate(f.Arguments[2]) is not { } c) + return null; + return build(AsInteger(a), AsInteger(b), AsInteger(c)); } - /// Access DateAdd(interval, number, date): add number intervals to a date. + /// + /// Access DateSerial (verified vs ACE): the month carries into the year first, then a year below 100 takes + /// its century from the calendar's two-digit year window (so 49 is 2049, 50 is 1950, -1 is 1999 and + /// DateSerial(100, 0, 1) is 1999-12-01), then the day carries. A year outside 100-9999 is an invalid procedure call. + /// + private static DateTime DateSerial(int year, int month, int day) + { + int months = year * 12 + month - 1; + year = (int)Math.Floor(months / 12.0); + month = months - year * 12 + 1; + if (year < 100) + { + int latest = CultureInfo.CurrentCulture.Calendar.TwoDigitYearMax; + int century = latest - latest % 100; + year += year <= latest % 100 ? century : century - 100; + } + return InDateRange(() => new DateTime(year, month, 1).AddDays(day - 1)); + } + + /// A computed date, or an invalid procedure call when it falls outside 100-01-01 … 9999-12-31. + private static DateTime InDateRange(Func compute) + { + try + { + DateTime date = compute(); + if (date.Year >= 100) + return date; + } + catch (ArgumentOutOfRangeException) + { + } + throw new ArgumentException("Invalid procedure call: the date is out of range."); + } + + /// + /// Access DateAdd(interval, number, date) (verified vs ACE): the number is truncated (True is -1), a month + /// step keeps the day where the month has it and takes the month's last day otherwise, and a result outside + /// 100-9999 is an invalid procedure call. "ms" is a LibRed extension. + /// private object? DateAdd(FunctionCall f) { - object? intervalV = Evaluate(f.Arguments[0]), numberV = Evaluate(f.Arguments[1]), dateV = Evaluate(f.Arguments[2]); - if (dateV is null || numberV is null) return null; - int n = (int)Math.Truncate(Convert.ToDouble(numberV, CultureInfo.InvariantCulture)); // Access truncates - var d = Convert.ToDateTime(dateV, CultureInfo.InvariantCulture); - return (intervalV?.ToString() ?? "").ToLowerInvariant() switch - { - "yyyy" => d.AddYears(n), - "q" => d.AddMonths(n * 3), - "m" => d.AddMonths(n), - "y" or "d" or "w" => d.AddDays(n), - "ww" => d.AddDays(n * 7), - "h" => d.AddHours(n), - "n" => d.AddMinutes(n), - "s" => d.AddSeconds(n), - "ms" => d.AddMilliseconds(n), - _ => throw new NotSupportedException($"DATEADD interval '{intervalV}' is not supported."), + if (Evaluate(f.Arguments[0]) is not { } interval || Evaluate(f.Arguments[1]) is not { } number + || Evaluate(f.Arguments[2]) is not { } date) + return null; + double n = Math.Truncate(Dbl(ConversionNumber(number))); + DateTime d = ToDate(date); + Func add = ConcatText(interval).ToLowerInvariant() switch + { + "yyyy" => () => d.AddYears(Int(n)), + "q" => () => d.AddMonths(checked(Int(n) * 3)), + "m" => () => d.AddMonths(Int(n)), + "y" or "d" or "w" => () => d.AddDays(n), + "ww" => () => d.AddDays(n * 7), + "h" => () => d.AddHours(n), + "n" => () => d.AddMinutes(n), + "s" => () => d.AddSeconds(n), + "ms" => () => d.AddMilliseconds(n), + _ => throw UnknownInterval(interval), }; + return InDateRange(add); } - /// Access DateDiff(interval, date1, date2): the number of interval boundaries from date1 to - /// date2 (a Long Integer). NULL-propagating. + /// + /// Access DateDiff(interval, date1, date2, [firstdayofweek], [firstweekofyear]) (verified vs ACE): the + /// number of interval boundaries from date1 to date2, as a Long Integer. "w" is whole weeks of days, "ww" counts + /// the weeks' first days, and "h", "n" and "s" count hour, minute and second boundaries; a count past a Long + /// Integer is an overflow. The first day of the week matters only to "ww", and the first week of the year to none. + /// private object? DateDiff(FunctionCall f) { - object? intervalV = Evaluate(f.Arguments[0]), d1V = Evaluate(f.Arguments[1]), d2V = Evaluate(f.Arguments[2]); - if (d1V is null || d2V is null) return null; - var d1 = Convert.ToDateTime(d1V, CultureInfo.InvariantCulture); - var d2 = Convert.ToDateTime(d2V, CultureInfo.InvariantCulture); - string interval = (intervalV?.ToString() ?? "").ToLowerInvariant(); + if (Evaluate(f.Arguments[0]) is not { } intervalV || Evaluate(f.Arguments[1]) is not { } d1V + || Evaluate(f.Arguments[2]) is not { } d2V) + return null; + DateTime d1 = ToDate(d1V), d2 = ToDate(d2V); + string interval = ConcatText(intervalV).ToLowerInvariant(); // "ms" is a LibRed extension — ACE's interval list stops at "s". It is available because LibRed stores // the full OA double rather than truncating to whole seconds as ACE does, and it is exact: .NET's OA @@ -1318,19 +1881,28 @@ private static object BitwiseOp(object a, object b, Func op) = return (long)(d2 - d1).TotalMilliseconds; } + if (interval == "ww") + { + if (FirstDayOfWeek(f, 3) is not { } first) + return null; + return (d2.Date.AddDays(-DaysIntoWeek(d2, first)) - d1.Date.AddDays(-DaysIntoWeek(d1, first))).Days / 7; + } + return interval switch { "yyyy" => d2.Year - d1.Year, "q" => (d2.Year - d1.Year) * 4 + (d2.Month - 1) / 3 - (d1.Month - 1) / 3, "m" => (d2.Year - d1.Year) * 12 + d2.Month - d1.Month, - "y" or "d" or "w" => (int)(d2.Date - d1.Date).TotalDays, - "ww" => (int)((d2.Date - d1.Date).TotalDays / 7), - "h" => (int)(d2 - d1).TotalHours, - "n" => (int)(d2 - d1).TotalMinutes, - "s" => (int)(d2 - d1).TotalSeconds, + "y" or "d" => (d2.Date - d1.Date).Days, + "w" => (d2.Date - d1.Date).Days / 7, + "h" => Boundaries(TimeSpan.TicksPerHour), + "n" => Boundaries(TimeSpan.TicksPerMinute), + "s" => Boundaries(TimeSpan.TicksPerSecond), // "ms" is handled above, as Int64. - _ => throw new NotSupportedException($"DATEDIFF interval '{intervalV}' is not supported."), + _ => throw UnknownInterval(intervalV), }; + + int Boundaries(long unit) => checked((int)(d2.Ticks / unit - d1.Ticks / unit)); } // Access truthiness: a filter/logical context treats any non-zero number as true (so a boolean stored @@ -1343,16 +1915,8 @@ private static object BitwiseOp(object a, object b, Func op) = return u.Operator switch { UnaryOperator.Not => AsBool(v) is bool b ? !b : null, // coerce a -1/0 integer boolean too - UnaryOperator.Negate => v switch // preserve the operand's numeric type (EF contract), like C# unary minus - { - null => null, - decimal d => -d, - double db => -db, - float f => -f, - long or ulong => -Lng(v), - _ => -Int(v), // int/short/byte → int - }, - UnaryOperator.BitNot => v is null ? null : v is long or ulong ? (object)~Lng(v) : ~Int(v), + UnaryOperator.Negate => v is null ? null : Negate(v), + UnaryOperator.BitNot => v is null ? null : BitNot(v), UnaryOperator.IsNull => v is null, UnaryOperator.IsNotNull => v is not null, _ => throw new NotSupportedException($"Unary operator {u.Operator}."), @@ -1374,36 +1938,104 @@ private static object BitwiseOp(object a, object b, Func op) = bool? l = AsBool(Evaluate(b.Left)); return l == true ? true : l | AsBool(Evaluate(b.Right)); } + // XOR and EQV are Null when either side is; IMP is True whenever its left side is False or its right side + // is True, and Null otherwise when a side is Null (the VBA truth tables; verified vs ACE). + if (b.Operator is BinaryOperator.Xor or BinaryOperator.Eqv or BinaryOperator.Imp) + { + bool? l = AsBool(Evaluate(b.Left)), r = AsBool(Evaluate(b.Right)); + return b.Operator switch + { + BinaryOperator.Xor => l ^ r, + BinaryOperator.Eqv => l is null || r is null ? null : l == r, + _ => l == false || r == true ? true : l is null || r is null ? null : false, + }; + } - object? left = Evaluate(b.Left); - object? right = Evaluate(b.Right); + // A date plus or less a span (IsSpan) is the date moved by it. Which side is the span is settled before + // either is evaluated, so neither side is worked out twice; beside anything but a date, the span side is + // evaluated as it always is — as the time on the epoch. + object? left, right; + if (SpanSide(b) is { } spanOnRight) + { + object? other = Evaluate(spanOnRight ? b.Left : b.Right); + if (other is DateTime date) + return SpanOf(spanOnRight ? b.Right : b.Left) is { } span + ? Shift(date, spanOnRight && b.Operator == BinaryOperator.Subtract ? -span : span) + : null; + (left, right) = spanOnRight ? (other, Evaluate(b.Right)) : (Evaluate(b.Left), other); + } + else + { + left = Evaluate(b.Left); + right = Evaluate(b.Right); + } + // '&' treats a single Null as "" but is Null when both sides are (verified vs ACE). if (b.Operator == BinaryOperator.Concat) - return (left is null ? "" : ToText(left)) + (right is null ? "" : ToText(right)); + return left is null && right is null + ? null + : (left is null ? "" : ConcatText(left)) + (right is null ? "" : ConcatText(right)); + + // The arithmetic operators other than '+' read text as a number even when the other side is Null, so text + // that is not a number, a GUID or a binary value is a type mismatch before Null propagates (verified vs + // ACE: 'abc' * NULL fails, '1' * NULL is Null). + if (b.Operator is BinaryOperator.Subtract or BinaryOperator.Multiply or BinaryOperator.Divide + or BinaryOperator.IntDivide or BinaryOperator.Modulo or BinaryOperator.Power) + { + left = NumericOperand(left); + right = NumericOperand(right); + } + + // A number written with a decimal point meets a Decimal as the value written, not as the Double it was + // parsed to (verified vs ACE: a DECIMAL(18,4) 4.5 * 334.90 is 1507.05, not 1507.04999…). + if (b.Operator is BinaryOperator.Add or BinaryOperator.Subtract or BinaryOperator.Multiply or BinaryOperator.Divide) + { + if (right is decimal && WrittenValue(b.Left) is decimal leftWritten) left = leftWritten; + if (left is decimal && WrittenValue(b.Right) is decimal rightWritten) right = rightWritten; + } + + // A pattern written as just '%' is False for Null rather than Null, so it reads as IS NOT NULL (verified vs ACE; + // '%%', or a '%' the pattern is computed to, stays Null). + if (left is null && b.Operator == BinaryOperator.Like && b.Right is LiteralExpression { Value: "%" }) + return false; if (left is null || right is null) return null; + if (b.Operator is BinaryOperator.Equal or BinaryOperator.NotEqual && TruthTest(b, left, right) is bool truth) + return b.Operator == BinaryOperator.Equal ? truth : !truth; + + // A result that is a Currency (NumberTypeOf) is one at every step, not only in the result column: its four + // places and its range apply to it where it is worked out, as CCur applies them (verified vs ACE: Currency + // 1.2345 * 1.2345 is 1.524, 0.0003 * 1.2345 is 0.0004, and the largest Currency + 1 or * 3 is an + // overflow). A Currency with a written decimal or a Double is not one, so those are left as they are. + if (b.Operator is BinaryOperator.Add or BinaryOperator.Subtract or BinaryOperator.Multiply) + { + object result = b.Operator switch + { + BinaryOperator.Add => Add(left, right), + BinaryOperator.Subtract => Arithmetic(left, right, '-'), + _ => Arithmetic(left, right, '*'), + }; + return result is decimal money && IsCurrency(b) ? ToCurrency(money) : result; + } + return b.Operator switch { - BinaryOperator.Equal => Compare(left, right) == 0, - BinaryOperator.NotEqual => Compare(left, right) != 0, - BinaryOperator.LessThan => Compare(left, right) < 0, - BinaryOperator.LessThanOrEqual => Compare(left, right) <= 0, - BinaryOperator.GreaterThan => Compare(left, right) > 0, - BinaryOperator.GreaterThanOrEqual => Compare(left, right) >= 0, - // LIKE reads a binary value as text, so it is CASE-INSENSITIVE over a binary column even though - // '=' on the same column is byte-wise and case-sensitive. Verified vs ACE: `B LIKE 'A%'` matches - // both 0x4100 ('A') and 0x6100 ('a'), while `B = 0x4100` matches only the first. - BinaryOperator.Like => Like(ToText(left), ToText(right)), - // Access '+' concatenates when either operand is text (but, unlike '&', null already propagated above). - BinaryOperator.Add => left is string || right is string ? ToText(left) + ToText(right) : Arithmetic(left, right, '+'), - BinaryOperator.Subtract => Arithmetic(left, right, '-'), - BinaryOperator.Multiply => Arithmetic(left, right, '*'), + BinaryOperator.Equal => CompareAsKinds(left, right) == 0, + BinaryOperator.NotEqual => CompareAsKinds(left, right) != 0, + BinaryOperator.LessThan => CompareAsKinds(left, right) < 0, + BinaryOperator.LessThanOrEqual => CompareAsKinds(left, right) <= 0, + BinaryOperator.GreaterThan => CompareAsKinds(left, right) > 0, + BinaryOperator.GreaterThanOrEqual => CompareAsKinds(left, right) >= 0, + // LIKE reads any other value as the text CStr gives it (verified vs ACE: TRUE LIKE '-1' is True). A binary + // value becomes text too, so LIKE is case-insensitive over a binary column even though '=' on the same + // column is byte-wise: `B LIKE 'A%'` matches both 0x4100 ('A') and 0x6100 ('a'). + BinaryOperator.Like => LikeMatcher.IsMatch(ConcatText(left), ConcatText(right)), BinaryOperator.Divide => Divide(left, right), // Access '/' is floating division BinaryOperator.Modulo => IntegerOp(left, right, '%'), BinaryOperator.IntDivide => IntegerOp(left, right, '\\'), - BinaryOperator.Power => Math.Pow(Convert.ToDouble(left, CultureInfo.InvariantCulture), Convert.ToDouble(right, CultureInfo.InvariantCulture)), + BinaryOperator.Power => Power(left, right), BinaryOperator.BitAnd => BitwiseOp(left, right, (x, y) => x & y), BinaryOperator.BitOr => BitwiseOp(left, right, (x, y) => x | y), BinaryOperator.BitXor => BitwiseOp(left, right, (x, y) => x ^ y), @@ -1411,60 +2043,203 @@ private static object BitwiseOp(object a, object b, Func op) = }; } - /// SQL LIKE: '%'/'*' match any run, '_'/'?' match one char; case-insensitive. - private static bool Like(string value, string pattern) + /// + /// Whether is a span — known before any row is read, as a Currency result is. + /// A span starts from a TimeSpan or TimeOnly parameter (); only a parameter can say it + /// is one, since a time column or a time written into the SQL is a date on the epoch, as Jet stores it. A span + /// stays one under negation, added to or less another span, times or divided by something that is not a span, + /// and chosen by IIF, COALESCE or CASE from spans and Nulls. A date plus or less a span is the date moved by it + /// () — a date, where a date less a date is a day count. + /// + internal static bool IsSpan(Expression expression, Func duration) { - // Access/Jet LIKE wildcards: * or % = any run, ? or _ = any single char, # = any single DIGIT, and - // [charlist] / [!charlist] = a (negated) single-char class. A literal special char is escaped by - // bracketing it — e.g. EF's Contains("C#") emits `%C[#]%`, where [#] matches a literal '#'. Without - // bracket-class support that pattern would look for the literal text "C[#]" and match nothing. - var sb = new StringBuilder("^"); - int i = 0; - while (i < pattern.Length) + bool Span(Expression e) => IsSpan(e, duration); + bool SpanOrNull(Expression e) => e is LiteralExpression { Value: null } || Span(e); + bool Choice(IEnumerable results) => results.All(SpanOrNull) && results.Any(Span); + + return expression switch { - char ch = pattern[i]; - if (ch == '[') + ParameterExpression parameter => duration(parameter.Name) is not null, + UnaryExpression { Operator: UnaryOperator.Negate } negation => Span(negation.Operand), + BinaryExpression { Operator: BinaryOperator.Add or BinaryOperator.Subtract } sum => Span(sum.Left) && Span(sum.Right), + BinaryExpression { Operator: BinaryOperator.Multiply } product => Span(product.Left) != Span(product.Right), + BinaryExpression { Operator: BinaryOperator.Divide } quotient => Span(quotient.Left) && !Span(quotient.Right), + FunctionCall call when call.Name.Equals("IIF", StringComparison.OrdinalIgnoreCase) && call.Arguments.Count >= 2 => + Choice(call.Arguments.Skip(1)), + FunctionCall call when call.Name.Equals("COALESCE", StringComparison.OrdinalIgnoreCase) => Choice(call.Arguments), + CaseExpression @case => Choice(@case.WhenClauses.Select(w => w.Result) + .Concat(@case.ElseResult is { } otherwise ? [otherwise] : [])), + _ => false, + }; + } + + /// Which side of a + or - is a span that a date beside it would move by: true for the right, + /// false for the left (only for +, since a span less a date is no date), null when neither. + private bool? SpanSide(BinaryExpression b) + { + if (parameters is null || b.Operator is not (BinaryOperator.Add or BinaryOperator.Subtract)) + return null; + if (IsSpan(b.Right, parameters.Duration)) + return true; + return b.Operator == BinaryOperator.Add && IsSpan(b.Left, parameters.Duration) ? false : null; + } + + /// The value of a span (), to the tick; null when it is Null. A factor or divisor is read + /// as a number as the arithmetic operators read one. + private TimeSpan? SpanOf(Expression expression) + { + switch (expression) + { + case ParameterExpression parameter: + return parameters!.Duration(parameter.Name); + case LiteralExpression { Value: null }: + return null; + case UnaryExpression negation: + return -SpanOf(negation.Operand); + case BinaryExpression { Operator: BinaryOperator.Add or BinaryOperator.Subtract } sum: + return SpanOf(sum.Left) is { } a && SpanOf(sum.Right) is { } b + ? checked(sum.Operator == BinaryOperator.Add ? a + b : a - b) + : null; + case BinaryExpression product: { - int close = pattern.IndexOf(']', i + 1); - if (close > i) - { - sb.Append(TranslateLikeClass(pattern.Substring(i + 1, close - i - 1))); - i = close + 1; - continue; - } - // No closing ']' → a literal '['. + bool spanOnLeft = product.Operator == BinaryOperator.Divide || IsSpan(product.Left, parameters!.Duration); + if (SpanOf(spanOnLeft ? product.Left : product.Right) is not { } span + || Evaluate(spanOnLeft ? product.Right : product.Left) is not { } by) + return null; + double factor = Dbl(ConversionNumber(by)); + if (product.Operator == BinaryOperator.Divide) + factor = factor != 0 ? 1 / factor : throw new DivideByZeroException("Division by zero."); + return TimeSpan.FromTicks(checked((long)Math.Round(span.Ticks * factor))); } + case FunctionCall call when call.Name.Equals("IIF", StringComparison.OrdinalIgnoreCase): + return Evaluate(call.Arguments[0]) is { } condition && IifCondition(condition) + ? SpanOf(call.Arguments[1]) + : call.Arguments.Count == 3 ? SpanOf(call.Arguments[2]) : null; + case FunctionCall coalesce: + foreach (Expression argument in coalesce.Arguments) + if (SpanOf(argument) is { } first) + return first; + return null; + case CaseExpression @case: + foreach (CaseWhen when in @case.WhenClauses) + if (IsTrue(when.Condition)) + return SpanOf(when.Result); + return @case.ElseResult is { } otherwise ? SpanOf(otherwise) : null; + default: + throw new InvalidOperationException($"{expression.GetType().Name} is not a span."); + } + } - sb.Append(ch switch - { - '%' or '*' => ".*", - '_' or '?' => ".", - '#' => "[0-9]", - _ => Regex.Escape(ch.ToString()), - }); - i++; + /// A date moved by a span, to the tick; outside the dates Jet holds, an overflow. + private static DateTime Shift(DateTime date, TimeSpan by) + { + long ticks = date.Ticks + by.Ticks; + return ticks >= new DateTime(100, 1, 1).Ticks && ticks <= DateTime.MaxValue.Ticks + ? new DateTime(ticks) + : throw new OverflowException($"Overflow: {date} moved by {by} is outside the range of a date."); + } + + /// + /// = or <> against the literal True or False tests the other side's truth + /// rather than comparing it (verified vs ACE): a value that reads as 0 is False and anything else is True, so + /// 2 = True, 'abc' = True and '' = True are all True, while '0' = False is True. + /// Null for any other comparison. + /// + private static bool? TruthTest(BinaryExpression b, object left, object right) => + b.Right is LiteralExpression { Value: bool rightLiteral } ? IsTruthy(left) == rightLiteral + : b.Left is LiteralExpression { Value: bool leftLiteral } ? IsTruthy(right) == leftLiteral + : null; + + /// + /// IIf's condition, which unlike the others is rounded to a whole number first, half to even (verified vs ACE: + /// 0.4, 0.5, '0.4' and a date at 06:00 on 1899-12-30 are False, 0.6 and 1E+20 are True). Text that is not a + /// number is True, as has it. + /// + private static bool IifCondition(object value) => value switch + { + bool b => b, + string or char => !(TryTextAsNumber(value.ToString()!) is { } number && Math.Round(number, MidpointRounding.ToEven) == 0), + DateTime d => Math.Round(d.ToOADate(), MidpointRounding.ToEven) != 0, + Guid or byte[] => true, + _ => !IsNumeric(value) || Math.Round(Dbl(value), MidpointRounding.ToEven) != 0, + }; + + /// Text read as reads it, or null when it is not a number or is past a Double. + private static double? TryTextAsNumber(string text) + { + try + { + return TextAsNumber(text); + } + catch (Exception e) when (e is InvalidCastException or OverflowException) + { + return null; } - sb.Append('$'); - return Regex.IsMatch(value, sb.ToString(), RegexOptions.IgnoreCase | RegexOptions.Singleline); } - /// Translates an Access LIKE bracket list (the text between [ and ]) to a regex - /// character class: a leading ! is negation (^), ranges (a-z) carry over, and the - /// regex-special \ ] ^ are escaped so a bracketed literal like [#]/[[] matches itself. - private static string TranslateLikeClass(string inner) + private static bool IsTruthy(object value) => value switch { - var sb = new StringBuilder("["); - if (inner.StartsWith('!')) { sb.Append('^'); inner = inner[1..]; } - foreach (char c in inner) + bool b => b, + string s => !ReadsAsZero(s), + char c => !ReadsAsZero(c.ToString()), + DateTime d => d != OaEpoch, + Guid or byte[] => true, + _ => !IsNumeric(value) || Dbl(value) != 0, + }; + + private static bool ReadsAsZero(string text) + { + try { - if (c is '\\' or ']' or '^') sb.Append('\\'); - sb.Append(c); + return TextAsDecimal(text) == 0m; + } + catch (InvalidCastException) + { + return false; // not a number, so not 0 } - sb.Append(']'); - return sb.ToString(); } - private static bool? AsBool(object? v) => v switch { bool b => b, null => null, _ => Convert.ToBoolean(v) }; + /// + /// The two sides of a comparison, brought to a kind they compare as (verified vs ACE, except as noted). + /// + /// Text against a number, Boolean or date: the text reads as a number (), a type + /// mismatch when it is not one. ACE does this for text a function returns; for a text literal or column it + /// raises a type mismatch instead, which LibRed does not, as SQL Server compares them numerically too. + /// A date against a number: its serial. + /// A GUID or binary value against text, or against the other of the two: both as text (a GUID braced upper + /// case, binary as UTF-16). Against a number or date: a type mismatch. + /// + /// + private static (object Left, object Right) Comparable(object left, object right) + { + if (left is char leftChar) left = leftChar.ToString(); + if (right is char rightChar) right = rightChar.ToString(); + bool leftText = left is string, rightText = right is string; + bool leftBlob = left is Guid or byte[], rightBlob = right is Guid or byte[]; + if ((leftText && rightBlob) || (leftBlob && rightText) || (leftBlob && rightBlob && left.GetType() != right.GetType())) + return (ConcatText(left), ConcatText(right)); + if (leftBlob || rightBlob) + return leftBlob && rightBlob + ? (left, right) + : throw new InvalidCastException("Type mismatch: a GUID or binary value cannot be compared with a number or date."); + if (leftText == rightText) + return left is DateTime ^ right is DateTime ? (Serial(left), Serial(right)) : (left, right); + return (leftText ? TextAsNumber((string)left) : Serial(left), rightText ? TextAsNumber((string)right) : Serial(right)); + } + + /// The order of two values once has brought them to a common kind. + private static int CompareAsKinds(object left, object right) + { + (object l, object r) = Comparable(left, right); + return Compare(l, r); + } + + private static object Serial(object value) => value is DateTime d ? d.ToOADate() : value; + + /// A value as a condition — for NOT, AND, OR, XOR, EQV, IMP and a + /// WHERE: False when it reads as 0, True otherwise, Null when Null (verified vs ACE: '0' is False, + /// 'abc', '', a date, a GUID and a binary value are True). + private static bool? AsBool(object? v) => v is null ? null : IsTruthy(v); /// + - * with C# widest-operand type promotion, so the result CLR type matches what EF /// expects (int+int→int, …): decimal > double > single > long > int. (Contract: like @@ -1482,63 +2257,519 @@ private static object Arithmetic(object left, object right, char op) return op switch { '-' when bothDates => a - b, // date − date → number of days - '+' => RoundToSecond(DateTime.FromOADate(a + b)), - '-' => RoundToSecond(DateTime.FromOADate(a - b)), - _ => a * b, // date × n has no date meaning → numeric + '+' => OaDate(a + b), + '-' => OaDate(a - b), + _ => Finite(a * b), // date × n has no date meaning → numeric }; } - if (left is decimal || right is decimal) { decimal a = Dec(left), b = Dec(right); return op == '+' ? a + b : op == '-' ? a - b : a * b; } - if (left is double || right is double) { double a = Dbl(left), b = Dbl(right); return op == '+' ? a + b : op == '-' ? a - b : a * b; } - if (left is float || right is float) { float a = (float)Dbl(left), b = (float)Dbl(right); return op == '+' ? a + b : op == '-' ? a - b : a * b; } - if (left is long or ulong || right is long or ulong) { long a = Lng(left), b = Lng(right); return op == '+' ? a + b : op == '-' ? a - b : a * b; } - int x = Int(left), y = Int(right); return op == '+' ? x + y : op == '-' ? x - y : x * y; + // A result past its type is an error, not a wrapped or infinite value (verified vs ACE: 2147483647 + 1, + // 2147483647 * 2, -2147483647 - 2 and 1E300 * 1E300 all fail). + if (left is decimal || right is decimal) { decimal a = ArithmeticDecimal(left), b = ArithmeticDecimal(right); return op == '+' ? a + b : op == '-' ? a - b : a * b; } + if (left is double || right is double) { double a = Dbl(left), b = Dbl(right); return op == '+' ? a + b : Finite(op == '-' ? a - b : a * b); } + if (left is float || right is float) { float a = (float)Dbl(left), b = (float)Dbl(right); return op == '+' ? a + b : Finite(op == '-' ? a - b : a * b); } + if (left is long or ulong || right is long or ulong) { long a = Lng(left), b = Lng(right); return checked(op == '+' ? a + b : op == '-' ? a - b : a * b); } + int x = Int(left), y = Int(right); return checked(op == '+' ? x + y : op == '-' ? x - y : x * y); } - /// Access / is floating division — Decimal when either operand is Decimal/Currency, - /// otherwise Double (never integer division; that is \). - private static object Divide(object left, object right) => - left is decimal || right is decimal ? Dec(left) / Dec(right) : Dbl(left) / Dbl(right); + /// The exact value of a number written with a decimal point (negated as written), or null. + private static decimal? WrittenValue(Expression expression) + { + bool negate = false; + while (expression is UnaryExpression { Operator: UnaryOperator.Negate } negation) + { + negate = !negate; + expression = negation.Operand; + } + return expression is LiteralExpression { Written: decimal written } ? (negate ? -written : written) : null; + } - /// Access integer operators \ (int division) and MOD: operands round to an - /// integer, and the result keeps the operand's integer type (int, or long if either is Int64) — so - /// int \ int is Int32, matching the EF contract. - private static object IntegerOp(object left, object right, char op) + /// + /// The type ACE gives a result column before reading any row, as far as its places go (verified vs ACE). ACE + /// works the expression out in full and cuts the value only as it goes into a Decimal result column — so + /// Pmt(0.05 / 12, …) uses the whole rate while SELECT 1 / 1.5 is 0.6. + /// + /// A number written with a decimal point is a Decimal of the places written less trailing zeros + /// (334.90 has one; 3.0 is a whole number), and so is a whole number too big for a Long, with + /// none; a Decimal column has its scale; Currency counts as four places. + /// * and /: a Decimal with whole numbers, dates, Booleans or text stays that Decimal + /// (DECIMAL(18,4) 4.5 / 7 is 0.6428; 1.5 / '2.5' is 0.6); two Decimals of the same places stay it (1.5 * 1.5 is + /// 2.2); two of different places, a Double or Single, or a Currency divided give a Double (1.5 * 1.25 is + /// 1.875; 1.5 / DECIMAL 4.5 is 0.333…). + /// + and - keep the same rule, except that Currency with a Decimal is a Decimal of the + /// larger places (1.5 - Currency 3.25 is -1.7500). + /// A function other than CCur and Sum/Min/Max/First/Last, a + /// parameter or anything else is not a Decimal; a function returning text or a date counts as text or a date + /// (1.5 / Left('12', 2) is 0.1), as says. + /// + /// + internal static NumberType NumberTypeOf( + Expression expression, IReadOnlyList columns, Func declaredType) { - if (left is long or ulong || right is long or ulong) - { long a = Lng(left), b = Lng(right); return op == '%' ? a % b : a / b; } - int x = Int(left), y = Int(right); return op == '%' ? x % y : x / y; + NumberType Of(Expression e) => NumberTypeOf(e, columns, declaredType); + + switch (expression) + { + case LiteralExpression { Written: decimal written }: + { + int places = (written / 1.0000000000000000000000000000m).Scale; // trailing zeros dropped + return places == 0 ? new(NumberClass.Whole) : new(NumberClass.Decimal, places); + } + case LiteralExpression literal: + return literal.Value switch + { + // A whole number too big for a Long is a Decimal to ACE, so Currency * 864000000000 is a Double + // there rather than an overflowing Currency (verified vs ACE); LibRed keeps the value an Int64. + long => new(NumberClass.Decimal, 0), + int or short or byte or bool => new(NumberClass.Whole), + double or float => new(NumberClass.Double), + string => new(NumberClass.Text), + DateTime => new(NumberClass.Date), + _ => new(NumberClass.Other), + }; + case ColumnReference reference: + if (OutputColumn.Find(columns, reference) is not { } column) + return new(NumberClass.Other); + if (column.Currency) + return new(NumberClass.Currency); + if (column.Scale is int scale) + return new(NumberClass.Decimal, scale); + Type? type = column.ClrType; + return type == typeof(double) || type == typeof(float) ? new(NumberClass.Double) + : type == typeof(int) || type == typeof(long) || type == typeof(short) || type == typeof(byte) + || type == typeof(bool) ? new(NumberClass.Whole) + : type == typeof(string) ? new(NumberClass.Text) + : type == typeof(DateTime) ? new(NumberClass.Date) + : new(NumberClass.Other); + case UnaryExpression { Operator: UnaryOperator.Negate } negation: + return Of(negation.Operand); + case BinaryExpression { Operator: BinaryOperator.Multiply or BinaryOperator.Divide } product: + return Product(Of(product.Left), Of(product.Right), product.Operator == BinaryOperator.Divide); + case BinaryExpression { Operator: BinaryOperator.Add or BinaryOperator.Subtract } sum: + return Sum(Of(sum.Left), Of(sum.Right), sum.Operator == BinaryOperator.Add); + case BinaryExpression { Operator: BinaryOperator.IntDivide or BinaryOperator.Modulo }: + return new(NumberClass.Whole); + case BinaryExpression { Operator: BinaryOperator.Power }: + return new(NumberClass.Double); + case FunctionCall function when function.Name.Equals("CCUR", StringComparison.OrdinalIgnoreCase): + return new(NumberClass.Currency); + case FunctionCall { Arguments: [var argument] } function + when function.Name.ToUpperInvariant() is "SUM" or "MIN" or "MAX" or "FIRST" or "LAST": + return Of(argument); + case FunctionCall { WithinGroup: not null, Arguments: [_, var key] } function + when function.Name.Equals("PERCENTILE_DISC", StringComparison.OrdinalIgnoreCase): + return Of(key); + case FunctionCall function: + Type? returns = declaredType(function); + return returns == typeof(string) ? new(NumberClass.Text) + : returns == typeof(DateTime) ? new(NumberClass.Date) + : new(NumberClass.Other); + default: + return new(NumberClass.Other); + } } - /// VBA CStr. A Double renders at 15 significant digits and a Single at 7 — the OA/VB - /// convention, not .NET Core 3.0+'s shortest-round-trippable form, which would turn 0.1+0.2 into - /// "0.30000000000000004" (verified vs ACE: "0.3", and CStr(CSng(1/3)) is "0.3333333"). A Boolean - /// renders as its VARIANT_BOOL number, "-1" — note that is the Jet Expression Service's behaviour and - /// differs from the VBA runtime proper, which renders "True". - private static string VbaString(object v) => v switch + private static bool IsWhole(NumberType type) => type.Class is NumberClass.Whole or NumberClass.Text or NumberClass.Date; + + private static NumberType Product(NumberType left, NumberType right, bool divide) + { + if (left.Class == NumberClass.Other || right.Class == NumberClass.Other) + return new(NumberClass.Other); + if (left.Class == NumberClass.Double || right.Class == NumberClass.Double + || divide && (left.Class == NumberClass.Currency || right.Class == NumberClass.Currency)) + return new(NumberClass.Double); + if (IsWhole(left) && IsWhole(right)) + return divide ? new(NumberClass.Double) : new(NumberClass.Whole); + if (IsWhole(left)) return right; + if (IsWhole(right)) return left; + if (left.Class == NumberClass.Currency && right.Class == NumberClass.Currency) + return new(NumberClass.Currency); + int leftPlaces = left.Class == NumberClass.Currency ? 4 : left.Places; + int rightPlaces = right.Class == NumberClass.Currency ? 4 : right.Places; + return leftPlaces == rightPlaces ? new(NumberClass.Decimal, leftPlaces) : new(NumberClass.Double); + } + + private static NumberType Sum(NumberType left, NumberType right, bool add) + { + if (left.Class is NumberClass.Other or NumberClass.Date || right.Class is NumberClass.Other or NumberClass.Date + || add && left.Class == NumberClass.Text && right.Class == NumberClass.Text) + return new(NumberClass.Other); + if (left.Class == NumberClass.Double || right.Class == NumberClass.Double) + return new(NumberClass.Double); + if (IsWhole(left) && IsWhole(right)) + return new(NumberClass.Whole); + if (IsWhole(left)) return right; + if (IsWhole(right)) return left; + if (left.Class == NumberClass.Currency && right.Class == NumberClass.Currency) + return new(NumberClass.Currency); + if (left.Class == NumberClass.Currency || right.Class == NumberClass.Currency) + return new(NumberClass.Decimal, Math.Max(4, Math.Max(left.Places, right.Places))); + return left.Places == right.Places ? left : new(NumberClass.Double); + } + + /// + /// A value as it goes into a result column of : a Decimal column keeps only its places, + /// the rest cut off. A Double or Single — LibRed's type for a written decimal literal — becomes a Decimal the + /// way OLE Automation converts it (), so binary rounding (0.6 held as + /// 0.5999…) cannot drop a digit. + /// + internal static object? ToResultPlaces(object? value, NumberType type) => + type.Class != NumberClass.Decimal ? value : value switch + { + decimal d => decimal.Round(d, type.Places, MidpointRounding.ToZero), + double or float => CutFloating(value, type.Places), + _ => value, + }; + + /// + /// A value converted to the type its result column declares, when it has another: a number to a wider number + /// (a Boolean as -1 or 0, a Double into a Decimal the OLE Automation way), anything to text as & writes + /// it, and anything to binary as its bytes (). Null, or no , leaves + /// the value as it is. + /// + internal static object? AsColumnType(object? value, Type? type, bool currency) + { + if (value is null || type is null || value.GetType() == type) + return value; + if (type == typeof(string)) return ConcatText(value); + if (type == typeof(byte[])) return ColumnBytes(value, currency); + if (type == typeof(decimal)) return Dec(value); + if (type == typeof(double)) return Dbl(value); + return Convert.ChangeType(Numeric(value), type, CultureInfo.InvariantCulture); + } + + /// + /// A value in a binary result column (verified vs ACE, which writes it so where a UNION mixes binary or GUID + /// values with others): text as UTF-16, a GUID as its 16 bytes, a Boolean as a 16-bit -1 or 0, a Byte or a Long + /// as 4 bytes, an Integer as 2, a Single as 4 and a Double or a date's serial as 8, Currency as its 8-byte scaled + /// integer, and a Decimal or a Large Number as its text. ACE cuts a Large Number's text to 8 bytes, which LibRed + /// does not. + /// + private static byte[] ColumnBytes(object value, bool currency) => value switch + { + byte[] bytes => bytes, + string text => Encoding.Unicode.GetBytes(text), + Guid guid => guid.ToByteArray(), + bool b => BitConverter.GetBytes((short)(b ? -1 : 0)), + byte b => BitConverter.GetBytes((int)b), + short s => BitConverter.GetBytes(s), + int i => BitConverter.GetBytes(i), + float f => BitConverter.GetBytes(f), + double d => BitConverter.GetBytes(d), + DateTime date => BitConverter.GetBytes(date.ToOADate()), + decimal m when currency => BitConverter.GetBytes(decimal.ToOACurrency(m)), + _ => Encoding.Unicode.GetBytes(ConcatText(value)), + }; + + private static object CutFloating(object value, int places) { - bool b => b ? "-1" : "0", - double d => d.ToString("G15", CultureInfo.InvariantCulture), - float f => f.ToString("G7", CultureInfo.InvariantCulture), - _ => Convert.ToString(v, CultureInfo.InvariantCulture)!, + try + { + decimal exact = value is float f ? JetDecimalConverter.FromSingle(f) : JetDecimalConverter.FromDouble((double)value); + decimal cut = decimal.Round(exact, places, MidpointRounding.ToZero); + return value is float ? (float)cut : (double)cut; + } + catch (OverflowException) + { + return value; // past a Decimal: ACE could not have held it as one either + } + } + + /// A number as a Decimal for arithmetic with a Decimal: a Double or Single converted the OLE Automation + /// way (), a date as its serial. + private static decimal ArithmeticDecimal(object v) => v is DateTime date ? SerialDecimal(date) : Dec(v); + + private static double Finite(double value) => + double.IsFinite(value) ? value : throw new OverflowException("Overflow: the result is too large for a number."); + + private static float Finite(float value) => + float.IsFinite(value) ? value : throw new OverflowException("Overflow: the result is too large for a number."); + + /// The date at an OLE Automation serial; a serial outside 100-01-01 … 9999-12-31 is an overflow. + private static DateTime OaDate(double serial) + { + try + { + return DateTime.FromOADate(serial); + } + catch (ArgumentException) + { + throw new OverflowException($"Overflow: {serial} is outside the range of a date."); + } + } + + /// A date's day number as a Decimal, built from its parts rather than converted from the Double serial + /// (before the epoch the time fraction still counts away from zero, as in the OLE Automation serial). + private static decimal SerialDecimal(DateTime d) + { + decimal days = (d.Date - OaEpoch).Days; + decimal time = d.TimeOfDay.Ticks / (decimal)TimeSpan.TicksPerDay; + return days >= 0 ? days + time : days - time; + } + + + /// + /// Access ^ (verified vs ACE): Double, a date read as its serial. A negative base with a fractional + /// exponent is an invalid procedure call, zero to a negative power a division by zero, and a result past a + /// Double an overflow. + /// + private static double Power(object left, object right) + { + double x = Oa(left), y = Oa(right); + double result = Math.Pow(x, y); + if (double.IsNaN(result)) + throw new ArgumentException($"Invalid procedure call: {x} cannot be raised to the power {y}."); + if (double.IsInfinity(result)) + throw x == 0 ? new DivideByZeroException("Division by zero: zero raised to a negative power.") : new OverflowException("Overflow: the result is too large for a number."); + return result; + } + + /// + /// Access + (verified vs ACE). Two text operands concatenate — a GUID or binary value counts as text. + /// Otherwise the operands add, text read as a number (); so '1' + 1 is 2 and + /// '1' + #2020-01-02# is the next day. A GUID or binary value with anything but text is a type + /// mismatch. Null has already propagated. + /// + private static object Add(object left, object right) => + IsConcatText(left) && IsConcatText(right) + ? ConcatText(left) + ConcatText(right) + : Arithmetic(NumericOperand(left)!, NumericOperand(right)!, '+'); + + private static bool IsConcatText(object v) => v is string or char or Guid or byte[]; + + /// An arithmetic operand: text read as a number (); a GUID or binary value + /// is a type mismatch; anything else, Null included, as it is. A (a parameter can carry one) + /// is one character of text. + private static object? NumericOperand(object? v) => v switch + { + string s => TextAsNumber(s), + char c => TextAsNumber(c.ToString()), + Guid or byte[] => throw new InvalidCastException("Type mismatch: a GUID or binary value is not a number."), + _ => v, }; - /// VBA CBool: any non-zero number is True (so 0.5 is True), and a string may hold a number - /// ("-1") as well as "True"/"False". rejects the numeric-string form - /// with a FormatException, so ACE accepts input LibRed used to refuse (verified vs ACE). - private static bool VbaBool(object v) => v switch + /// + /// A value as & (and + between two texts) writes it (verified vs ACE). A Boolean is its + /// VARIANT_BOOL number; a Double has 15 significant digits and a Single 7; a Decimal drops trailing zeros; a + /// date is written in the regional short date and long time, without the time at midnight and without the + /// date on 1899-12-30; a GUID is braced upper case; a binary value is read as UTF-16 text. + /// + internal static string ConcatText(object v) => v switch { - bool b => b, - string s => bool.TryParse(s, out var parsed) - ? parsed - : double.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var n) - ? n != 0 - : throw new InvalidOperationException($"Type mismatch: '{s}' cannot be converted to Boolean."), - _ => Dbl(v) != 0, + string s => s, + bool b => b ? "-1" : "0", + double d => FloatingText(d, 15), + float f => FloatingText(f, 7), + decimal m => m.ToString("0.############################", CultureInfo.CurrentCulture), + DateTime d => DateText(d), + Guid g => g.ToString("B").ToUpperInvariant(), + byte[] => Encoding.Unicode.GetString(ToBytes(v)), + _ => Convert.ToString(v, CultureInfo.CurrentCulture)!, }; + /// + /// A floating value rounded to significant digits, trailing zeros dropped. Written + /// in E notation when its exponent is at least , or when fixed notation would need + /// more than decimals (verified vs ACE: 1/3 is 0.333333333333333, a Single + /// 1E7 is 1E+07, a Single 1E-5 is 0.00001, 1E300 is 1E+300). + /// + private static string FloatingText(double value, int digits, NumberFormatInfo? format = null) + { + if (value == 0) return "0"; + if (!double.IsFinite(value)) return value.ToString(CultureInfo.InvariantCulture); + + format ??= CultureInfo.CurrentCulture.NumberFormat; + string scientific = value.ToString("E" + (digits - 1), CultureInfo.InvariantCulture); // -d.dddE+ddd + int mark = scientific.IndexOf('E'); + int exponent = int.Parse(scientific[(mark + 1)..], CultureInfo.InvariantCulture); + bool negative = scientific[0] == '-'; + string significant = scientific[(negative ? 1 : 0)..mark].Replace(".", "").TrimEnd('0'); + string sign = negative ? format.NegativeSign : ""; + string point = format.NumberDecimalSeparator; + + int decimals = significant.Length - 1 - exponent; + if (exponent >= digits || decimals > digits) + { + string mantissa = significant.Length == 1 ? significant : significant[0] + point + significant[1..]; + return $"{sign}{mantissa}E{(exponent < 0 ? "-" : "+")}{Math.Abs(exponent):00}"; + } + if (exponent < 0) + return sign + "0" + point + new string('0', -exponent - 1) + significant; + if (significant.Length <= exponent + 1) + return sign + significant + new string('0', exponent + 1 - significant.Length); + return sign + significant[..(exponent + 1)] + point + significant[(exponent + 1)..]; + } + + private static readonly DateTime OaEpoch = new(1899, 12, 30); + + /// The culture's long time pattern, with the narrow no-break space newer ICU data puts before AM/PM + /// (U+202F, on Linux and macOS) written as the plain space of Windows' regional settings, and so of ACE. A date + /// then writes the same text on every platform. + private static string LongTimePattern(DateTimeFormatInfo format) => format.LongTimePattern.Replace(' ', ' '); + + /// A date as writes it; the year is not zero-padded (year 100 is "100") unless + /// asks for it, as Format does. + private static string DateText(DateTime d, bool padYear = false) + { + DateTimeFormatInfo format = CultureInfo.CurrentCulture.DateTimeFormat; + string time = d.ToString(LongTimePattern(format), CultureInfo.CurrentCulture); + if (d.Date == OaEpoch) return time; + string datePattern = padYear + ? format.ShortDatePattern + : format.ShortDatePattern.Replace("yyyy", "'" + d.Year.ToString(CultureInfo.InvariantCulture) + "'"); + string date = d.ToString(datePattern, CultureInfo.CurrentCulture); + return d.TimeOfDay == TimeSpan.Zero ? date : date + " " + time; + } + + /// + /// Text as + reads it as a number, in the regional separators and currency symbol (verified vs ACE). + /// Surrounding whitespace is skipped. One sign may come before or after the number, spaced from it; brackets + /// make it negative but cannot be combined with a sign; one currency symbol may come on either side. Group + /// separators are ignored once a digit has been read, even after the decimal point. An e or d + /// exponent needs at least one digit. &H/&O text is a whole number, read as a Long when + /// it fits 32 bits (&HFFFFFFFF is -1). Anything else is a type mismatch, and a value past a Double + /// is an overflow. + /// + private static double TextAsNumber(string text) + { + double value = double.Parse(NumberText(text), NumberStyles.Float, CultureInfo.InvariantCulture); + if (double.IsInfinity(value)) + throw new OverflowException($"Overflow: '{text}' is too large for a number."); + return value; + } + + /// Text read as reads it, but exactly; null when it is past a Decimal. + private static decimal? TextAsDecimal(string text) => + decimal.TryParse(NumberText(text), NumberStyles.Float, CultureInfo.InvariantCulture, out decimal value) ? value : null; + + /// The number reads, written out in invariant form. + private static string NumberText(string text) + { + NumberFormatInfo format = CultureInfo.CurrentCulture.NumberFormat; + string s = text.Trim(); + if (s.Length > 1 && s[0] == '&' && s[1] is 'H' or 'h' or 'O' or 'o') + return RadixNumber(text, s[2..], s[1] is 'H' or 'h' ? 16 : 8).ToString(CultureInfo.InvariantCulture); + + int i = 0; + bool negative = false, signed = false, currency = false, open = false, closed = false; + while (i < s.Length) + { + if (char.IsWhiteSpace(s[i])) i++; + else if (s[i] is '+' or '-' && !signed) { signed = true; negative = s[i] == '-'; i++; } + else if (s[i] == '(' && !open) { open = true; i++; } + else if (!currency && At(s, i, format.CurrencySymbol)) { currency = true; i += format.CurrencySymbol.Length; } + else break; + } + + var number = new StringBuilder(); + bool digits = false, point = false; + while (i < s.Length) + { + if (s[i] is >= '0' and <= '9') { number.Append(s[i]); digits = true; i++; } + else if (!point && At(s, i, format.NumberDecimalSeparator)) { number.Append('.'); point = true; i += format.NumberDecimalSeparator.Length; } + else if (digits && At(s, i, format.NumberGroupSeparator)) i += format.NumberGroupSeparator.Length; + else break; + } + if (!digits) throw NotANumber(text); + if (point && number[^1] == '.') number.Append('0'); + + if (i < s.Length && s[i] is 'e' or 'E' or 'd' or 'D') + { + number.Append('E'); + i++; + if (i < s.Length && s[i] is '+' or '-') number.Append(s[i++]); + int start = i; + while (i < s.Length && s[i] is >= '0' and <= '9') number.Append(s[i++]); + if (i == start) throw NotANumber(text); + } + + while (i < s.Length) + { + if (char.IsWhiteSpace(s[i])) i++; + else if (s[i] is '+' or '-' && !signed) { signed = true; negative = s[i] == '-'; i++; } + else if (s[i] == ')' && open && !closed) { closed = true; i++; } + else if (!currency && At(s, i, format.CurrencySymbol)) { currency = true; i += format.CurrencySymbol.Length; } + else throw NotANumber(text); + } + if (open != closed || (open && signed)) throw NotANumber(text); + + return negative || open ? "-" + number : number.ToString(); + } + + private static long RadixNumber(string text, string digits, int radix) + { + if (digits.Length == 0) throw NotANumber(text); + ulong value = 0; + foreach (char c in digits) + { + int digit = c is >= '0' and <= '9' ? c - '0' + : c is >= 'A' and <= 'F' ? c - 'A' + 10 + : c is >= 'a' and <= 'f' ? c - 'a' + 10 + : int.MaxValue; + if (digit >= radix) throw NotANumber(text); + value = checked(value * (ulong)radix + (ulong)digit); + } + return value <= uint.MaxValue ? (int)(uint)value : checked((long)value); + } + + private static bool At(string s, int index, string symbol) => + symbol.Length > 0 && string.CompareOrdinal(s, index, symbol, 0, symbol.Length) == 0; + + private static InvalidCastException NotANumber(string text) => + new($"Type mismatch: '{text}' cannot be read as a number."); + + /// Access / is floating division — Decimal when either operand is Decimal/Currency, + /// otherwise Double (never integer division; that is \). A date divides as its serial; dividing by + /// zero is an error and so is a result past a Double (verified vs ACE: 1 / 0 and 0 / 0 fail). + private static object Divide(object left, object right) + { + if (left is decimal || right is decimal) + return ArithmeticDecimal(left) / ArithmeticDecimal(right); + // A Single divided with only Singles, Integers or Booleans is a Single (verified vs ACE: TRUE / a Single 1.5 + // is -0.6666667, and Str(CSng(1) / CSng(10)) is " .1"). OLE DB reports it as a Double, as it does CSng. + if ((left is float || right is float) && IsSingleWidth(left) && IsSingleWidth(right)) + { + float singleDivisor = Sng(right); + if (singleDivisor == 0) + throw new DivideByZeroException("Division by zero."); + return Finite(Sng(left) / singleDivisor); + } + double divisor = Oa(right); + if (divisor == 0) + throw new DivideByZeroException("Division by zero."); + return Finite(Oa(left) / divisor); + } + + // Not Byte: ACE treats a Byte column as a Long here (verified: a Byte 1 / a Single 1.5 is 0.6666666666666666). + private static bool IsSingleWidth(object v) => v is float or short or bool; + + /// Access integer operators \ (int division) and MOD: operands round to an + /// integer (half to even), a date as its serial, and the result keeps the operand's integer type (int, or + /// long if either is Int64) — so int \ int is Int32, matching the EF contract. Anything MOD -1 is 0, + /// even the smallest Long (verified vs ACE). + /// An Int32 result is worked out in Int64 all the same, and only the result has to fit: ACE squeezes + /// each operand into a Long first, so a Double, Decimal or Currency past one overflows even where the answer + /// would not — 1E12 MOD 7, whose remainder is below 7. Here that is 1, and a quotient past a Long is still + /// an overflow, since the column's type is settled before any value is seen. A LibRed extension. + private static object IntegerOp(object left, object right, char op) + { + left = Serial(left); + right = Serial(right); + long a = Lng(left), b = Lng(right); + long result = op == '%' ? (b == -1 ? 0L : a % b) : a / b; + // Each branch boxed on its own: a bare `? result : (int)result` is a long, and would widen the Int32 back. + return left is long or ulong || right is long or ulong ? (object)result : checked((int)result); + } + + /// VBA CBool (verified vs ACE): "True" and "False" as written, otherwise whether the value + /// read as a number () is non-zero, so 0.5, '$5' and a date are True. + private static bool VbaBool(object v) => + v is bool b ? b + : v is string s && bool.TryParse(s, out bool parsed) ? parsed + : Dbl(ConversionNumber(v)) != 0; + // Jet's boolean convention (true = -1, false = 0) so a bool matches the numeric column it is stored in. private static object Numeric(object v) => v is bool b ? (b ? -1 : 0) : v; - private static decimal Dec(object v) => Convert.ToDecimal(Numeric(v), CultureInfo.InvariantCulture); + private static decimal Dec(object v) => JetDecimalConverter.ToDecimal(Numeric(v), CultureInfo.InvariantCulture); private static double Dbl(object v) => Convert.ToDouble(Numeric(v), CultureInfo.InvariantCulture); // Narrow to single precision (the cast yields ±Infinity for an out-of-range double rather than throwing). private static float Sng(object v) => (float)Dbl(v); @@ -1548,16 +2779,6 @@ private static object IntegerOp(object left, object right, char op) // For date arithmetic: a DateTime becomes its OLE Automation serial; a number is taken verbatim (as days). private static double Oa(object v) => v is DateTime d ? d.ToOADate() : Dbl(v); - // Rounds to the nearest whole second — Jet stores no sub-second, and the OA-serial round-trip can leave - // a value a few ticks shy of/past an exact second (e.g. 21:05:18.9999999). - private static DateTime RoundToSecond(DateTime d) - { - long rem = d.Ticks % TimeSpan.TicksPerSecond; - return rem >= TimeSpan.TicksPerSecond - rem - ? d.AddTicks(TimeSpan.TicksPerSecond - rem) - : d.AddTicks(-rem); - } - private static int Compare(object left, object right) { if (IsNumeric(left) && IsNumeric(right)) @@ -1600,8 +2821,16 @@ private static int Compare(object left, object right) // while the index compares by serial made an index seek and a table scan return DIFFERENT rows for a // pre-epoch range (see PreEpochDateOrderingTests). From the epoch onward the two orders are identical, // so this only affects pre-1899 dates. + // + // ToOADate keeps whole milliseconds, which is all a Date/Time column holds, but a DATETIME2 keeps 100-ns + // ticks: two of those in the same millisecond are settled by their ticks, the way the serial would run — + // later first below the epoch — so they neither compare equal nor sort as a tie. if (left is DateTime leftDate && right is DateTime rightDate) - return leftDate.ToOADate().CompareTo(rightDate.ToOADate()); + { + int bySerial = leftDate.ToOADate().CompareTo(rightDate.ToOADate()); + return bySerial != 0 ? bySerial + : leftDate.Ticks.CompareTo(rightDate.Ticks) * (leftDate < OaEpoch ? -1 : 1); + } if (left is IComparable c && left.GetType() == right.GetType()) return c.CompareTo(right); @@ -1642,13 +2871,13 @@ private static int CompareBytes(byte[] a, byte[] b) return a.Length.CompareTo(b.Length); } - /// Access text comparison: **case-insensitive**, **trailing spaces ignored**, and - /// **accent-aware ordering** (Access "General" collation). Uses invariant-culture ignore-case, which - /// matches ACE where ordinal doesn't — an accented letter sorts next to its base letter (verified vs - /// ACE: 'é' < 'f', 'café' < 'cafz') while accents stay significant for equality - /// ('café' ≠ 'cafe'). (Ignorable apostrophe/hyphen ordering is not reproduced.) + /// Access text comparison, in the database sort order (): case-insensitive, + /// trailing spaces ignored, an accented letter beside its base letter but not equal to it (verified vs ACE: + /// 'é' < 'f', 'café' ≠ 'cafe'), 'ß' = 'ss', and a hyphen weighed after the letters. A + /// character that order does not cover compares case-insensitively. private static int CompareText(string a, string b) => - string.Compare(a.TrimEnd(' '), b.TrimEnd(' '), StringComparison.InvariantCultureIgnoreCase); + JetTextComparer.Compare(a, b) + ?? Math.Sign(string.Compare(a.TrimEnd(' '), b.TrimEnd(' '), StringComparison.InvariantCultureIgnoreCase)); /// Orders two values for SORT (nulls first), using the same coercion as comparisons. public static int CompareForSort(object? a, object? b) => (a, b) switch diff --git a/src/LibRed/LibRed.Engine/Execution/JoinPredicateHoisting.cs b/src/LibRed/LibRed.Engine/Execution/JoinPredicateHoisting.cs index a50f95eb1..1320362fe 100644 --- a/src/LibRed/LibRed.Engine/Execution/JoinPredicateHoisting.cs +++ b/src/LibRed/LibRed.Engine/Execution/JoinPredicateHoisting.cs @@ -76,30 +76,9 @@ void Walk(Expression e) return; } - switch (e) + foreach (Expression operand in e.Operands() ?? []) { - case BinaryExpression b: - Walk(b.Left); - Walk(b.Right); - break; - case UnaryExpression u: - Walk(u.Operand); - break; - case FunctionCall f: - foreach (Expression a in f.Arguments) - { - Walk(a); - } - - break; - case InListExpression il: - Walk(il.Value); - foreach (Expression i in il.Items) - { - Walk(i); - } - - break; + Walk(operand); } } } @@ -112,21 +91,7 @@ void Walk(Expression e) internal static Expression Substitute(Expression on, IReadOnlyDictionary values) => values.TryGetValue(on, out object? value) ? new LiteralExpression(value) - : on switch - { - BinaryExpression b => b with - { - Left = Substitute(b.Left, values), - Right = Substitute(b.Right, values), - }, - UnaryExpression u => u with { Operand = Substitute(u.Operand, values) }, - FunctionCall f => f with { Arguments = f.Arguments.Select(a => Substitute(a, values)).ToList() }, - InListExpression il => il with - { - Value = Substitute(il.Value, values), - Items = il.Items.Select(i => Substitute(i, values)).ToList(), - }, - InSubqueryExpression isq => isq with { Value = Substitute(isq.Value, values) }, - _ => on, - }; + : on is InSubqueryExpression isq + ? isq with { Value = Substitute(isq.Value, values) } + : on.MapOperands(o => Substitute(o, values)); } diff --git a/src/LibRed/LibRed.Engine/Execution/LikeMatcher.cs b/src/LibRed/LibRed.Engine/Execution/LikeMatcher.cs new file mode 100644 index 000000000..84858487c --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/LikeMatcher.cs @@ -0,0 +1,191 @@ +namespace LibRed.Engine.Execution; + +/// +/// The LIKE match with ANSI-92 wildcards, the set EF Core emits over both OLE DB and ODBC (verified vs ACE). +/// +/// +/// +/// % matches any run and _ any one character. *, ? and # are plain +/// characters. +/// [abc] matches one of the characters listed, [!abc] one character not listed, and a-c +/// inside the brackets a range. A - at either end is itself listed, [!] is the character !, and +/// [] matches nothing at all, so []] is a plain ] and [[]] the text []. The +/// Access documentation gives ^ as the ANSI-92 negation, but ACE does not treat it so: [^ae] lists +/// ^, a and e, and ! negates as it does in ANSI-89. +/// Case is ignored and accents are not. ß counts as ss and æ as ae, in the pattern, +/// in a bracket list and in the value, so 'aßb' LIKE 'a[s]sb' is True; _ still takes the whole +/// character. +/// A bracket that is never closed, or a range written backwards, is an invalid pattern. It is only reported +/// when the match reaches it with a character left to test: '' LIKE '[' and 'z' LIKE 'x[z-a]' are +/// both False. +/// There is no ESCAPE clause; ACE rejects it as a syntax error. A wildcard is matched literally by +/// bracketing it. +/// +/// Documentation: https://support.microsoft.com/en-us/office/like-operator-b2f7ef03-9085-4ffb-9829-eef18358e931, +/// https://support.microsoft.com/en-us/office/access-wildcard-character-reference-af00c501-7972-40ee-8889-e18abaad12d1 +/// and https://support.microsoft.com/en-us/office/use-wildcards-in-queries-and-parameters-in-access-ec057a45-78b1-4d16-8c20-242cde582e0b. +/// +internal static class LikeMatcher +{ + /// The two characters a character counts as (ß as ss, æ and Æ as ae), or null for itself. + private static string? Expansion(char c) => c switch + { + 'ß' => "ss", + 'æ' => "ae", + 'Æ' => "AE", + _ => null, + }; + + public static bool IsMatch(string value, string pattern) + { + string text = Fold(value); + int[] characterEnd = CharacterEnds(value, text.Length); + + // reach[p] is whether the pattern so far can consume exactly the first p characters of the text. + var reach = new bool[text.Length + 1]; + reach[0] = true; + int i = 0; + while (i < pattern.Length) + { + int first = Array.IndexOf(reach, true); + if (first < 0) + return false; + + if (pattern[i] == '%') + { + Array.Fill(reach, true, first, reach.Length - first); + i++; + continue; + } + + var next = new bool[reach.Length]; + if (pattern[i] == '_') + { + // One character of the value, however many it folds to (verified vs ACE: 'ß' LIKE '_' is True). + for (int p = first; p < text.Length; p++) + next[characterEnd[p]] |= reach[p]; + i++; + } + else if (pattern[i] == '[') + { + int close = pattern.IndexOf(']', i + 1); + if (close < 0) + return Invalid(first, text); + if (close == i + 1) + { + i += 2; // [] matches nothing + continue; + } + if (BracketList.Parse(pattern.AsSpan(i + 1, close - i - 1)) is not { } list) + return Invalid(first, text); + for (int p = first; p < text.Length; p++) + { + if (reach[p]) + list.Match(text, p, next); + } + i = close + 1; + } + else + { + int end = pattern.IndexOfAny(['%', '_', '['], i); + if (end < 0) + end = pattern.Length; + string run = Fold(pattern[i..end]); + for (int p = first; p + run.Length <= text.Length; p++) + { + if (reach[p] && text.AsSpan(p).StartsWith(run, StringComparison.Ordinal)) + next[p + run.Length] = true; + } + i = end; + } + reach = next; + } + return reach[text.Length]; + } + + /// For each position of the folded value, where the value's character it belongs to ends. + private static int[] CharacterEnds(string value, int foldedLength) + { + var ends = new int[foldedLength]; + int p = 0; + foreach (char c in value) + { + int end = p + (Expansion(c)?.Length ?? 1); + for (; p < end; p++) + ends[p] = end; + } + return ends; + } + + private static bool Invalid(int first, string text) => + first < text.Length ? throw new ArgumentException("Invalid pattern string.") : false; + + /// Text as the match compares it: expansions spelt out, then upper case. Upper-casing keeps the length, + /// so positions in the folded text line up with one another. + private static string Fold(string text) + { + var folded = new System.Text.StringBuilder(text.Length); + foreach (char c in text) + { + if (Expansion(c) is { } expansion) + folded.Append(expansion); + else + folded.Append(c); + } + return folded.ToString().ToUpperInvariant(); + } + + private sealed class BracketList(bool negated, List members, List<(char Low, char High)> ranges) + { + /// The list between the brackets, or null when a range is written backwards. + public static BracketList? Parse(ReadOnlySpan body) + { + bool negated = body.Length > 1 && body[0] == '!'; + if (negated) + body = body[1..]; + + var members = new List(); + var ranges = new List<(char, char)>(); + for (int i = 0; i < body.Length; i++) + { + if (i + 2 < body.Length && body[i + 1] == '-') + { + if (body[i] > body[i + 2]) + return null; + ranges.Add((char.ToUpperInvariant(body[i]), char.ToUpperInvariant(body[i + 2]))); + i += 2; + } + else + { + members.Add(Fold(body[i].ToString())); + } + } + return new BracketList(negated, members, ranges); + } + + /// Marks in each position the list can advance to. A + /// member that expands matches two characters, where anything else matches one. + public void Match(string text, int p, bool[] next) + { + bool any = false; + foreach (string member in members) + { + if (text.AsSpan(p).StartsWith(member, StringComparison.Ordinal)) + { + any = true; + next[p + member.Length] |= !negated; + } + } + foreach ((char low, char high) in ranges) + { + if (text[p] >= low && text[p] <= high) + { + any = true; + next[p + 1] |= !negated; + } + } + if (negated && !any) + next[p + 1] = true; + } + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/ListAgg.cs b/src/LibRed/LibRed.Engine/Execution/ListAgg.cs new file mode 100644 index 000000000..ab13c93b9 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/ListAgg.cs @@ -0,0 +1,42 @@ +using LibRed.Sql.Ast; + +namespace LibRed.Engine.Execution; + +/// +/// The standard's LISTAGG(x [, separator]) WITHIN GROUP (ORDER BY …): the non-Null values of a group or a +/// window frame as text, in the WITHIN GROUP order, joined by the separator (none when it is left out). Access has no +/// such aggregate; this is a LibRed extension. +/// +internal static class ListAgg +{ + /// + /// The list over — each a value and its WITHIN GROUP key values — or Null when no value + /// is present. Each value is written as & writes it. Rows whose keys tie keep their order. Under + /// a value repeated — equal as GROUP BY takes values to be equal — is listed once, + /// where it first comes. + /// + public static string? Of( + IEnumerable<(object? Value, object?[] Keys)> rows, string separator, IReadOnlyList directions, + bool distinct) + { + var comparer = Comparer.Create((a, b) => + { + for (int k = 0; k < directions.Count; k++) + { + int c = ExpressionEvaluator.CompareForSort(a[k], b[k]); + if (c != 0) + return directions[k] == SortDirection.Descending ? -c : c; + } + return 0; + }); + var values = rows.Where(r => r.Value is not null).OrderBy(r => r.Keys, comparer).Select(r => r.Value!); + if (distinct) + { + var seen = new HashSet(); + values = values.Where(v => seen.Add(new QueryExecutor.GroupKey([v]))); + } + + var text = values.Select(ExpressionEvaluator.ConcatText).ToList(); + return text.Count == 0 ? null : string.Join(separator, text); + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/ParameterBag.cs b/src/LibRed/LibRed.Engine/Execution/ParameterBag.cs index 95ba09504..5cd9fb7f4 100644 --- a/src/LibRed/LibRed.Engine/Execution/ParameterBag.cs +++ b/src/LibRed/LibRed.Engine/Execution/ParameterBag.cs @@ -5,8 +5,17 @@ namespace LibRed.Engine.Execution; /// case-insensitively and the leading @ is optional on both the reference and the key, /// so a value supplied as "p0" satisfies a "@p0" reference and vice versa. /// +/// +/// A or resolves to the Jet stores a time as — +/// the time on the OLE epoch, 1899-12-30 — so everything that reads a parameter, saving it included, still sees only +/// a date, as the evaluator expects. What it was is kept too (), because a date plus or less a +/// span is a date, where a date less a date is a day count. +/// internal sealed class ParameterBag { + /// The OLE epoch: Jet stores a time as the epoch plus the time of day. + private static readonly DateTime OleEpoch = new(1899, 12, 30); + private readonly Dictionary _values = new(StringComparer.OrdinalIgnoreCase); public ParameterBag(IReadOnlyDictionary? values) @@ -21,9 +30,32 @@ public ParameterBag(IReadOnlyDictionary? values) if (name.StartsWith('?')) throw new NotSupportedException("Positional ('?') parameters are not supported yet; use named (@name) parameters."); if (_values.TryGetValue(Normalize(name), out object? value)) - return value; + return value switch + { + TimeSpan span => OleEpoch + span, + TimeOnly time => OleEpoch + time.ToTimeSpan(), + _ => value, + }; throw new InvalidOperationException($"No value was supplied for parameter '{name}'."); } + /// The span a or parameter was bound to; null for any other + /// value, an unbound name or a positional reference. + public TimeSpan? Duration(string name) => + name.StartsWith('?') || !_values.TryGetValue(Normalize(name), out object? value) ? null + : value switch + { + TimeSpan span => span, + TimeOnly time => time.ToTimeSpan(), + _ => null, + }; + + /// The type of the value resolves to; null when it is unbound, positional or Null, + /// which declares nothing. + public Type? TypeOf(string name) => + name.StartsWith('?') || !_values.TryGetValue(Normalize(name), out object? value) || value is null ? null + : value is TimeSpan or TimeOnly ? typeof(DateTime) + : value.GetType(); + private static string Normalize(string name) => name.TrimStart('@'); } diff --git a/src/LibRed/LibRed.Engine/Execution/Percentile.cs b/src/LibRed/LibRed.Engine/Execution/Percentile.cs new file mode 100644 index 000000000..9fb047cd4 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/Percentile.cs @@ -0,0 +1,72 @@ +using System.Globalization; +using EntityFrameworkCore.Jet.Data; +using LibRed.Sql.Ast; + +namespace LibRed.Engine.Execution; + +/// +/// The standard's inverse distribution functions, PERCENTILE_CONT and PERCENTILE_DISC: the value at a +/// fraction of the way through the ordered non-Null values of a group or a window frame. Access has neither; this is a +/// LibRed extension. +/// +internal static class Percentile +{ + /// The declared type: PERCENTILE_DISC returns one of the values, so has the ORDER BY key's type; + /// PERCENTILE_CONT interpolates, giving a Double, or a date for dates. + public static Type? ResultType(string name, Type? key) => + name.Equals("PERCENTILE_DISC", StringComparison.OrdinalIgnoreCase) ? key + : key == typeof(DateTime) ? typeof(DateTime) + : typeof(double); + + /// + /// (upper case) over — the group's or frame's ORDER BY key values, + /// Nulls included, in any order — at . Null when there are no values or the fraction is + /// Null; a fraction outside 0 to 1 is an invalid procedure call. + /// + /// + /// PERCENTILE_DISC is the first value, in the given order, whose cumulative share of the values reaches the + /// fraction. PERCENTILE_CONT is at position fraction × (n − 1), interpolating linearly between the values + /// either side of it; text cannot be interpolated. The fraction is read as the Decimal it was written as, so + /// that 0.7 of 10 values is the 7th rather than the 8th, as a Double's 7.000000000000001 would make it. + /// + public static object? Of(string name, IEnumerable values, object? fraction, SortDirection direction) + { + if (fraction is null) + return null; + decimal p = JetDecimalConverter.ToDecimal(ExpressionEvaluator.ConversionNumber(fraction), CultureInfo.InvariantCulture); + if (p is < 0 or > 1) + throw new ArgumentException($"Invalid procedure call: a percentile fraction must be from 0 to 1, not {p}."); + + // A stable sort, so values that compare equal but differ — 'a' and 'A' — keep their order. + var comparer = Comparer.Create(ExpressionEvaluator.CompareForSort); + var present = values.Where(v => v is not null); + List sorted = direction == SortDirection.Descending + ? [.. present.OrderByDescending(v => v, comparer)] + : [.. present.OrderBy(v => v, comparer)]; + int n = sorted.Count; + if (n == 0) + return null; + + if (name == "PERCENTILE_DISC") + return sorted[Math.Max((int)Math.Ceiling(p * n), 1) - 1]; + + decimal position = p * (n - 1); + int below = (int)decimal.Floor(position); + int above = (int)decimal.Ceiling(position); + double share = (double)(position - below); + object low = sorted[below]!, high = sorted[above]!; + if (low is DateTime from && high is DateTime to) + { + // Along the timeline, to the whole second — dates carry no less. + long ticks = from.Ticks + (long)Math.Round((to.Ticks - from.Ticks) * share); + const long second = TimeSpan.TicksPerSecond; + return new DateTime((ticks + second / 2) / second * second); + } + double a = Number(low), b = Number(high); + return below == above ? a : a + (b - a) * share; + } + + private static double Number(object value) => value is string or char or Guid or byte[] + ? throw new InvalidCastException("Type mismatch: PERCENTILE_CONT interpolates numbers and dates.") + : Convert.ToDouble(ExpressionEvaluator.ConversionNumber(value), CultureInfo.InvariantCulture); +} diff --git a/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs b/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs index a9b805e8c..da5724aa6 100644 --- a/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs +++ b/src/LibRed/LibRed.Engine/Execution/QueryExecutor.cs @@ -4,8 +4,42 @@ namespace LibRed.Engine.Execution; -/// A column produced by a plan node: an optional table-alias qualifier and a name. -internal readonly record struct OutputColumn(string? Qualifier, string Name, Type? ClrType = null); +/// A column produced by a plan node: an optional table-alias qualifier and a name. +/// marks a Currency value, which shares with Decimal but calculates differently, and +/// a Decimal's places. marks a column that is a bare NULL, which +/// has no type of its own, unlike one whose type is merely unknown. +internal readonly record struct OutputColumn( + string? Qualifier, string Name, Type? ClrType = null, bool Currency = false, int? Scale = null, bool Null = false) +{ + /// The output of a stored column. + public static OutputColumn Of(string? qualifier, LibRed.Catalog.ColumnDef column) => + new(qualifier, column.Name, Schema.JetClrTypeMap.ToClrType(column.Type), + column.Type == LibRed.Catalog.JetDataType.Currency, + column.Type == LibRed.Catalog.JetDataType.FixedPoint ? column.Scale : null); + + /// A computed column of , computed by . + public static OutputColumn Computed(string name, Type? clrType, NumberType type, Expression expression) => + new(null, name, clrType, type.Class == NumberClass.Currency, + type.Class == NumberClass.Decimal ? type.Places : null, expression is LiteralExpression { Value: null }); + + /// The column names, or null when none or more than one does (execution + /// reports the ambiguous reference). + public static OutputColumn? Find(IReadOnlyList columns, ColumnReference reference) + { + OutputColumn? result = null; + foreach (OutputColumn column in columns) + { + if (!string.Equals(column.Name, reference.Column, StringComparison.OrdinalIgnoreCase) + || reference.Table is not null + && !string.Equals(column.Qualifier, reference.Table, StringComparison.OrdinalIgnoreCase)) + continue; + + if (result is not null) return null; + result = column; + } + return result; + } +} /// /// Interprets a logical plan tree against the storage layer, producing a @@ -453,8 +487,7 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) { var table = _database.OpenTable(scan.Table); string alias = scan.Alias ?? scan.Table; - var columns = table.Definition.Columns - .Select(c => new OutputColumn(alias, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(); + var columns = table.Definition.Columns.Select(c => OutputColumn.Of(alias, c)).ToList(); return (columns, table.Rows()); } @@ -462,8 +495,7 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) { var table = _database.OpenTable(seek.Table); string alias = seek.Alias ?? seek.Table; - var columns = table.Definition.Columns - .Select(c => new OutputColumn(alias, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(); + var columns = table.Definition.Columns.Select(c => OutputColumn.Of(alias, c)).ToList(); // Evaluate the key(s) in the outer scope (so an index-nested-loop join can key off the outer // row); a single-table seek's key is a constant/parameter. @@ -479,8 +511,7 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) { var table = _database.OpenTable(range.Table); string alias = range.Alias ?? range.Table; - var columns = table.Definition.Columns - .Select(c => new OutputColumn(alias, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(); + var columns = table.Definition.Columns.Select(c => OutputColumn.Of(alias, c)).ToList(); var evaluator = new ExpressionEvaluator(new EvalScope([], [], outer), this, parameters: _parameters, session: _session); int col = range.Index.Columns[0].Column.Index; @@ -497,7 +528,7 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) case DerivedTableNode derived: { var (inner, rows) = Execute(derived.Input, outer); - var columns = inner.Select(c => new OutputColumn(derived.Alias, c.Name, c.ClrType)).ToList(); + var columns = inner.Select(c => c with { Qualifier = derived.Alias }).ToList(); return (columns, rows); } @@ -546,7 +577,10 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) var projected = rows.Select(row => { var eval = Eval(columns, row, outer); - return plan.Select(p => p.InputIndex >= 0 ? row[p.InputIndex] : eval.Evaluate(p.Expr!)).ToArray(); + return plan.Select(p => p.InputIndex >= 0 + ? row[p.InputIndex] + : ExpressionEvaluator.ToResultPlaces( + ExpressionEvaluator.AsColumnType(eval.Evaluate(p.Expr!), p.ConvertTo, currency: false), p.Type)).ToArray(); }); return (schema.Columns, projected); @@ -554,10 +588,15 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) case SetOperationNode setOp: { - // Column names come from the left (leading) query, per SQL. - var (columns, leftRows) = Execute(setOp.Left, outer); - var (_, rightRows) = Execute(setOp.Right, outer); - return (columns, ExecuteSetOp(setOp.Operator, leftRows, rightRows)); + // Column names come from the left (leading) query, per SQL; each column's type is the one both + // queries' values fit, and every value is converted to it before rows are compared. + var (leftColumns, leftRows) = Execute(setOp.Left, outer); + var (rightColumns, rightRows) = Execute(setOp.Right, outer); + if (leftColumns.Count != rightColumns.Count) + return (leftColumns, ExecuteSetOp(setOp.Operator, leftRows, rightRows)); + var columns = leftColumns.Zip(rightColumns, SetOperationColumn).ToList(); + return (columns, ExecuteSetOp(setOp.Operator, + ToColumnTypes(leftRows, leftColumns, columns), ToColumnTypes(rightRows, rightColumns, columns))); } case LimitNode limit: @@ -634,6 +673,58 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) } } + /// + /// A set operation's column: the left query's, typed as ACE types it (verified vs ACE). A bare NULL takes + /// the other query's type. Numbers widen on , a Boolean counting as an Integer + /// (-1); a GUID or binary value with anything else makes a binary column; any other mix — text, or a date with a + /// number or a Boolean — makes a text column. An unknown type on either side leaves the column untyped. + /// + private static OutputColumn SetOperationColumn(OutputColumn left, OutputColumn right) + { + if (right.Null) + return left; + if (left.Null) + return left with { ClrType = right.ClrType, Currency = right.Currency, Scale = right.Scale, Null = false }; + + // A Decimal column is Currency unless one side is a Decimal of its own, and keeps a scale both sides share. + bool leftDecimal = left.ClrType == typeof(decimal) && !left.Currency; + bool rightDecimal = right.ClrType == typeof(decimal) && !right.Currency; + Type? type = left.ClrType is not { } l || right.ClrType is not { } r ? null + : l == r ? l + : l == typeof(Guid) || r == typeof(Guid) || l == typeof(byte[]) || r == typeof(byte[]) ? typeof(byte[]) + : CommonNumericType(AsInteger(l), AsInteger(r), currency: !leftDecimal && !rightDecimal) ?? typeof(string); + bool isDecimal = type == typeof(decimal); + return left with + { + ClrType = type, + Currency = isDecimal && !leftDecimal && !rightDecimal, + Scale = !isDecimal ? null + : leftDecimal && rightDecimal ? (left.Scale == right.Scale ? left.Scale : null) + : leftDecimal ? left.Scale : rightDecimal ? right.Scale : null, + }; + + static Type AsInteger(Type type) => type == typeof(bool) ? typeof(short) : type; + } + + /// The rows with each value converted to its output column's type, where the query's own column had + /// another. + private static IEnumerable ToColumnTypes( + IEnumerable rows, IReadOnlyList from, IReadOnlyList to) + { + int[] changed = Enumerable.Range(0, to.Count) + .Where(i => to[i].ClrType is { } type && !from[i].Null && from[i].ClrType != type) + .ToArray(); + if (changed.Length == 0) + return rows; + return rows.Select(row => + { + var converted = (object?[])row.Clone(); + foreach (int i in changed) + converted[i] = ExpressionEvaluator.AsColumnType(converted[i], to[i].ClrType, from[i].Currency); + return converted; + }); + } + private static IEnumerable ExecuteSetOp(SetOperator op, IEnumerable left, IEnumerable right) { switch (op) @@ -684,9 +775,11 @@ private PlanNode SubqueryPlan(SqlStatement query, EvalScope outerScope) /// null for expressions whose result type depends on runtime coercion; the ADO layer can still fall back /// to a non-null runtime value in those cases without publishing misleading metadata for empty results. /// A ProjectNode's flattened output: the per-item plan (source input index, or an expression to - /// evaluate) and the resulting output columns. Structural — the same for every outer row. + /// evaluate, with the type its values are converted to, if any) and the resulting output columns. + /// Structural — the same for every outer row. private sealed record ProjectionSchema( - List<(OutputColumn Column, int InputIndex, Expression? Expr)> Plan, List Columns); + List<(OutputColumn Column, int InputIndex, Expression? Expr, NumberType Type, Type? ConvertTo)> Plan, + List Columns); /// Builds — or reuses — a ProjectNode's schema. Flattens the projection, expanding a qualified star /// (Table.*) into the input columns of that source (passed through by index); every other item is an @@ -696,19 +789,21 @@ private ProjectionSchema ProjectionSchemaFor(ProjectNode project, IReadOnlyList< if (_projectionSchemas.TryGetValue(project, out ProjectionSchema? cached)) return cached; - var plan = new List<(OutputColumn Column, int InputIndex, Expression? Expr)>(); + var plan = new List<(OutputColumn Column, int InputIndex, Expression? Expr, NumberType Type, Type? ConvertTo)>(); foreach (SelectItem item in project.Projection) { if (item.Value is QualifiedStarExpression star) { for (int ci = 0; ci < columns.Count; ci++) if (string.Equals(columns[ci].Qualifier, star.Table, StringComparison.OrdinalIgnoreCase)) - plan.Add((columns[ci], ci, null)); + plan.Add((columns[ci], ci, null, default, null)); } else { string name = item.Alias ?? (item.Value is ColumnReference c ? c.Column : $"Expr{plan.Count + 1}"); - plan.Add((new OutputColumn(null, name, DeclaredType(item.Value, columns)), -1, item.Value)); + NumberType type = ExpressionEvaluator.NumberTypeOf(item.Value, columns, e => DeclaredType(e, columns)); + plan.Add((OutputColumn.Computed(name, DeclaredType(item.Value, columns), type, item.Value), -1, item.Value, + type, ChoiceConversion(item.Value, columns))); } } @@ -725,6 +820,11 @@ private ProjectionSchema ProjectionSchemaFor(ProjectNode project, IReadOnlyList< return value.GetType(); case LiteralExpression: return null; + // A parameter is typed by the value bound to it, as a literal is by its own — so `d + @p` declares the + // date it returns rather than nothing, which left the reader to guess from the first row. The value is + // the one the engine sees: the ADO layer has already turned a TimeSpan or TimeOnly into a DateTime. + case ParameterExpression parameter: + return _parameters.TypeOf(parameter.Name); case ColumnReference column: return DeclaredColumnType(column, columns) ?? (column.Table is null && column.Column.Equals("Now", StringComparison.OrdinalIgnoreCase) @@ -732,21 +832,27 @@ private ProjectionSchema ProjectionSchemaFor(ProjectNode project, IReadOnlyList< case SystemVariableExpression variable: return variable.Name.Equals("ROWCOUNT", StringComparison.OrdinalIgnoreCase) ? typeof(int) : _session?.LastIdentity?.GetType(); - case ExistsExpression or InSubqueryExpression or InListExpression: + case ExistsExpression or InSubqueryExpression or InListExpression or BetweenExpression: return typeof(bool); case UnaryExpression unary: return unary.Operator is UnaryOperator.Not or UnaryOperator.IsNull or UnaryOperator.IsNotNull - ? typeof(bool) : DeclaredType(unary.Operand, columns); + ? typeof(bool) : DeclaredUnaryType(unary.Operator, DeclaredType(unary.Operand, columns)); case BinaryExpression binary: if (binary.Operator is BinaryOperator.Equal or BinaryOperator.NotEqual or BinaryOperator.LessThan or BinaryOperator.LessThanOrEqual or BinaryOperator.GreaterThan or BinaryOperator.GreaterThanOrEqual - or BinaryOperator.And or BinaryOperator.Or or BinaryOperator.Like or BinaryOperator.In) + or BinaryOperator.And or BinaryOperator.Or or BinaryOperator.Xor or BinaryOperator.Eqv + or BinaryOperator.Imp or BinaryOperator.Like or BinaryOperator.In) return typeof(bool); if (binary.Operator == BinaryOperator.Concat) return typeof(string); Type? left = DeclaredType(binary.Left, columns); Type? right = DeclaredType(binary.Right, columns); + // A date plus or less a span is a date — ExpressionEvaluator.IsSpan, which the evaluator moves it by. + if (left == typeof(DateTime) && IsSpan(binary.Right) + && binary.Operator is BinaryOperator.Add or BinaryOperator.Subtract + || right == typeof(DateTime) && IsSpan(binary.Left) && binary.Operator == BinaryOperator.Add) + return typeof(DateTime); return DeclaredBinaryType(binary.Operator, left, right); case FunctionCall function: return DeclaredFunctionType(function, columns); @@ -762,7 +868,7 @@ or BinaryOperator.GreaterThan or BinaryOperator.GreaterThanOrEqual /// result_expressions and the optional else_result_expression". A branch whose own type is unknown /// contributes nothing rather than poisoning the answer, which is what makes a bare NULL arm /// harmless: a NULL literal has no type and the standard ignores it for precedence too. Numeric branches - /// widen along the same ladder as arithmetic, so THEN 1 ELSE 2.5 declares decimal. A genuine mix + /// widen on , so THEN 1 ELSE 2.5 declares Double. A genuine mix /// (a string branch and a numeric one) declares nothing rather than guessing, leaving the column untyped /// exactly as it was before CASE was understood at all. /// @@ -777,101 +883,185 @@ private static IEnumerable CaseResults(CaseExpression c) yield return c.ElseResult; } - /// The single type a set of alternative expressions declares — shared by CASE and COALESCE, - /// which the standard defines in terms of CASE and gives the same precedence rule. + /// The single type a set of alternative expressions declares — shared by CASE, IIF, COALESCE, + /// GREATEST and LEAST, which the standard defines in terms of CASE or gives the same precedence rule. + /// A number written with a decimal point counts as a Decimal here, as ACE reads it (NumberTypeOf), though + /// its value is held as a Double: EF writes a decimal zero as 0.0, so its Sum-with-a-default shape + /// IIF(SUM(x) IS NULL, 0.0, SUM(x)) would otherwise make every money sum a Double. A Double still wins + /// where one is present. private Type? UnifiedType(IEnumerable alternatives, IReadOnlyList columns) { Type? result = null; + bool currency = false; // whether a Decimal result so far is a Currency foreach (Expression alternative in alternatives) { - Type? branchType = DeclaredType(alternative, columns); + Type? branchType = IsWrittenDecimal(alternative) ? typeof(decimal) : DeclaredType(alternative, columns); if (branchType is null) continue; + bool branchCurrency = branchType == typeof(decimal) + && ExpressionEvaluator.NumberTypeOf(alternative, columns, e => DeclaredType(e, columns)).Class + == NumberClass.Currency; - if (result is null) + if (result is null || result == branchType) { + currency = result is null ? branchCurrency : currency && branchCurrency; result = branchType; continue; } - if (result == branchType) - continue; - - result = WidenNumeric(result, branchType); + bool decimalIsCurrency = result == typeof(decimal) ? currency : branchCurrency; + result = CommonNumericType(result, branchType, decimalIsCurrency); if (result is null) return null; + currency = result == typeof(decimal) && decimalIsCurrency; } return result; } - /// The wider of two numeric types, on the same ladder uses for - /// arithmetic. Null when either side is not numeric, meaning the two cannot be reconciled. - private static Type? WidenNumeric(Type left, Type right) + /// Whether is a number written with a decimal point, negated or not. + private static bool IsWrittenDecimal(Expression expression) + { + while (expression is UnaryExpression { Operator: UnaryOperator.Negate } negation) + expression = negation.Operand; + return expression is LiteralExpression { Written: decimal }; + } + + /// + /// The type an expression that picks one of several alternatives () converts its value + /// to, so the value has the type the column declares; null when nothing is converted. Only when every + /// alternative's type is known is the declared type sure to hold each of them. + /// + private Type? ChoiceConversion(Expression expression, IReadOnlyList columns) + { + IEnumerable? alternatives = expression switch + { + CaseExpression @case => CaseResults(@case), + FunctionCall function => function.Name.TrimEnd('$').ToUpperInvariant() switch + { + "IIF" when function.Arguments.Count == 3 => function.Arguments.Skip(1), + "COALESCE" or "GREATEST" or "LEAST" => function.Arguments, + _ => null, + }, + _ => null, + }; + if (alternatives is null + || alternatives.Any(a => a is not LiteralExpression { Value: null } && DeclaredType(a, columns) is null)) + return null; + return DeclaredType(expression, columns); + } + + /// + /// The type the values of two numeric types share (verified vs ACE, as it types a UNION): the wider whole + /// number of the two; a Single with a Byte or an Integer, and a Double for a Single with anything wider; a + /// Decimal (or Currency) with a whole number, and a Double with a Double. A Currency (, + /// when the Decimal side is one) cannot hold a Large Number, so the two make a Double. A Decimal with a Single, + /// which was not measured, is a Double too. Null when either side is not a number, meaning the two cannot be + /// reconciled. + /// + private static Type? CommonNumericType(Type left, Type right, bool currency = false) { if (!IsNumeric(left) || !IsNumeric(right)) return null; - if (left == typeof(decimal) || right == typeof(decimal)) return typeof(decimal); + if (left == right) return left; if (left == typeof(double) || right == typeof(double)) return typeof(double); - if (left == typeof(float) || right == typeof(float)) return typeof(float); - if (IsInt64(left) || IsInt64(right)) return typeof(long); - return typeof(int); + int leftRank = WholeRank(left), rightRank = WholeRank(right); + if (left == typeof(decimal) || right == typeof(decimal)) + return Math.Max(leftRank, rightRank) is var whole && whole < 0 || currency && whole > 2 + ? typeof(double) : typeof(decimal); + if (left == typeof(float) || right == typeof(float)) + return Math.Max(leftRank, rightRank) <= 1 ? typeof(float) : typeof(double); + return WholeTypes[Math.Max(leftRank, rightRank)]; } + private static readonly Type[] WholeTypes = [typeof(byte), typeof(short), typeof(int), typeof(long)]; + + /// A whole number type's place in (the narrowest that holds it), or -1. + private static int WholeRank(Type type) => + type == typeof(byte) ? 0 + : type == typeof(sbyte) || type == typeof(short) ? 1 + : type == typeof(ushort) || type == typeof(int) ? 2 + : type == typeof(uint) || IsInt64(type) ? 3 + : -1; + private static bool IsNumeric(Type type) => type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int) || type == typeof(uint) || IsInt64(type) || type == typeof(float) || type == typeof(double) || type == typeof(decimal); - private static Type? DeclaredColumnType(ColumnReference reference, IReadOnlyList columns) - { - Type? result = null; - bool found = false; - foreach (OutputColumn column in columns) - { - if (!string.Equals(column.Name, reference.Column, StringComparison.OrdinalIgnoreCase) - || reference.Table is not null - && !string.Equals(column.Qualifier, reference.Table, StringComparison.OrdinalIgnoreCase)) - continue; + private static Type? DeclaredColumnType(ColumnReference reference, IReadOnlyList columns) => + OutputColumn.Find(columns, reference)?.ClrType; - if (found) return null; // execution will report the ambiguous reference - found = true; - result = column.ClrType; - } - return result; - } + /// The declared type of an aggregate (upper-case name) over an argument of — + /// for an ordered-set aggregate, its WITHIN GROUP key — grouped or windowed. Keep in lock-step with + /// and . + internal static Type? AggregateResultType(string name, Type? argument) => RunningAggregate.Canonical(name) switch + { + "LISTAGG" => typeof(string), + "PERCENTILE_CONT" or "PERCENTILE_DISC" => Percentile.ResultType(name, argument), + "COUNT" or "REGR_COUNT" => typeof(int), + var pair when RunningAggregate.IsPair(pair) => typeof(double), + "SUM" => argument == null ? null + : argument == typeof(decimal) || argument == typeof(float) || argument == typeof(long) ? argument + : argument == typeof(ulong) ? typeof(long) + : argument == typeof(double) || argument == typeof(string) || argument == typeof(DateTime) ? typeof(double) + : typeof(int), + "AVG" => argument == typeof(decimal) ? typeof(decimal) : typeof(double), + "VAR" or "VARP" or "STDEV" or "STDEVP" or "STDDEV" or "STDDEVP" => typeof(double), + _ => argument, // MIN, MAX, FIRST and LAST keep the argument's type + }; private Type? DeclaredFunctionType(FunctionCall function, IReadOnlyList columns) { string name = function.Name.TrimEnd('$').ToUpperInvariant(); - Type? argument = function.Arguments.Count > 0 ? DeclaredType(function.Arguments[0], columns) : null; + Type? argument = function.Arguments.Count == 0 ? null + : DeclaredType(function.Arguments[function.WithinGroup is null ? 0 : ^1], columns); + if (QueryPlanner.IsAggregate(name)) + return AggregateResultType(name, argument); return name switch { - "COUNT" => typeof(int), - "SUM" or "MIN" or "MAX" or "FIRST" or "LAST" => argument, - "AVG" => argument == typeof(decimal) ? typeof(decimal) : typeof(double), "CBOOL" or "ISDATE" => typeof(bool), "CBYTE" => typeof(byte), "CINT" => typeof(short), "CLNG" => typeof(int), + "CLNGLNG" => typeof(long), "CSNG" => typeof(float), "CDBL" => typeof(double), "CDEC" or "CCUR" => typeof(decimal), + // Round, Abs, Int and Fix keep their operand's type — ExpressionEvaluator.Round and Numeric1. Undeclared, + // a Round beside a whole number in IIF, CASE or COALESCE left the Integer to declare the column while + // the Round arm returned a Decimal. + "ROUND" or "ABS" => KeptNumberType(argument, typeof(double)), + "INT" or "FIX" or "FLOOR" or "CEILING" or "CEIL" => KeptNumberType(argument, typeof(DateTime)), + "SGN" or "SIGN" => typeof(int), "CSTR" or "FORMAT" or "LCASE" or "UCASE" or "TRIM" or "LTRIM" or "RTRIM" or "LEFT" or "RIGHT" or "MID" or "REPLACE" or "STRING" or "SPACE" or "HEX" or "OCT" or "WEEKDAYNAME" or "MONTHNAME" or "PARTITION" => typeof(string), - "LEN" or "INSTR" or "INSTRREV" or "ASC" or "ASCW" or "DATEPART" or "DATEDIFF" + // DateDiff's "ms", LibRed's own interval, counts in Int64 — a millisecond difference passes Int32 after + // 25 days — where every other interval is a Long Integer. Only a written interval says which: one read + // from a parameter or a column is declared as the Long Integer the rest give. + "DATEDIFF" => function.Arguments is [LiteralExpression { Value: string interval }, ..] + && interval.Equals("ms", StringComparison.OrdinalIgnoreCase) + ? typeof(long) + : typeof(int), + "LEN" or "DATALENGTH" or "INSTR" or "INSTRREV" or "ASC" or "ASCW" or "DATEPART" or "YEAR" or "MONTH" or "DAY" or "HOUR" or "MINUTE" or "SECOND" or "WEEKDAY" => typeof(int), "CDATE" or "NOW" or "DATE" or "TIME" or "DATEADD" or "DATESERIAL" or "TIMESERIAL" or "DATEVALUE" or "TIMEVALUE" => typeof(DateTime), "SQR" or "SIN" or "COS" or "TAN" or "ATN" or "LOG" or "EXP" or "RND" - or "PMT" or "FV" or "PV" or "NPER" or "IPMT" or "PPMT" or "DDB" or "RATE" => typeof(double), - "IIF" when function.Arguments.Count == 3 => SameType( - DeclaredType(function.Arguments[1], columns), DeclaredType(function.Arguments[2], columns)), + or "SQRT" or "LN" or "LOG10" or "POWER" or "ASIN" or "ACOS" or "ATAN" or "ATAN2" or "SINH" or "COSH" + or "TANH" or "DEGREES" or "RADIANS" or "PI" + or "PMT" or "FV" or "PV" or "NPER" or "IPMT" or "PPMT" or "DDB" or "RATE" or "SLN" or "SYD" => typeof(double), + // IIF chooses between two values as CASE does, so it takes CASE's rule rather than ACE's own (which + // makes every whole number a Long and lets Currency beat Double). + "IIF" when function.Arguments.Count == 3 => UnifiedType(function.Arguments.Skip(1), columns), // The standard makes COALESCE shorthand for a CASE over its arguments, so it takes the same rule: // the highest-precedence type among them. Unified the same way, which also means a bare NULL // argument contributes no type rather than erasing the others. "COALESCE" => UnifiedType(function.Arguments, columns), + // GREATEST/LEAST return one of their arguments, so they declare the type the arguments unify to — + // SQL Server's "highest precedence type" rule, the same as COALESCE. + "GREATEST" or "LEAST" => UnifiedType(function.Arguments, columns), // NULLIF returns its first expression, or a NULL of that expression's type — so unlike COALESCE // it takes the first argument's type outright rather than unifying across both. The second // argument only ever participates in the comparison. @@ -880,19 +1070,47 @@ private static bool IsNumeric(Type type) }; } - private static Type? SameType(Type? left, Type? right) => left == right ? left : null; + + private bool IsSpan(Expression expression) => ExpressionEvaluator.IsSpan(expression, _parameters.Duration); + + /// The type Round, Abs, Int and Fix give over an operand of : a Decimal, Double, + /// Single or Int64 as it is, a narrower integer or a Boolean as a Long Integer, text as a Double, and a date as + /// . (Abs of the smallest Long Integer is the one value past a Long, and a Double.) + private static Type? KeptNumberType(Type? operand, Type whenDate) => + operand == typeof(decimal) || operand == typeof(double) || operand == typeof(float) || operand == typeof(long) + ? operand + : operand == typeof(int) || operand == typeof(short) || operand == typeof(byte) || operand == typeof(bool) + ? typeof(int) + : operand == typeof(DateTime) ? whenDate + : operand == typeof(string) || operand == typeof(char) ? typeof(double) + : null; private static Type? DeclaredBinaryType(BinaryOperator op, Type? left, Type? right) { if (left is null || right is null) return null; - if (op == BinaryOperator.Add && (left == typeof(string) || right == typeof(string))) + // '+' concatenates only two texts (a GUID or binary value counts as text). Otherwise an arithmetic + // operator reads text as a Double. Keep in lock-step with ExpressionEvaluator.Add and NumericOperand. + if (op == BinaryOperator.Add && IsConcatText(left) && IsConcatText(right)) return typeof(string); + if (op is BinaryOperator.Add or BinaryOperator.Subtract or BinaryOperator.Multiply or BinaryOperator.Divide + or BinaryOperator.IntDivide or BinaryOperator.Modulo or BinaryOperator.Power) + { + if (left == typeof(string)) left = typeof(double); + if (right == typeof(string)) right = typeof(double); + } if (op == BinaryOperator.Power) return typeof(double); + // Keep in lock-step with ExpressionEvaluator.Divide: a Single with only Singles, Integers or Booleans stays one. if (op == BinaryOperator.Divide) - return left == typeof(decimal) || right == typeof(decimal) ? typeof(decimal) : typeof(double); + return left == typeof(decimal) || right == typeof(decimal) ? typeof(decimal) + : (left == typeof(float) || right == typeof(float)) && IsSingleWidth(left) && IsSingleWidth(right) ? typeof(float) + : typeof(double); + // Keep in lock-step with ExpressionEvaluator.BitwiseOp: two 16-bit operands give an Integer. + if (op is BinaryOperator.BitAnd or BinaryOperator.BitOr or BinaryOperator.BitXor + && IsSixteenBits(left) && IsSixteenBits(right)) + return typeof(short); if (op is BinaryOperator.Modulo or BinaryOperator.IntDivide or BinaryOperator.BitAnd or BinaryOperator.BitOr or BinaryOperator.BitXor) return IsInt64(left) || IsInt64(right) ? typeof(long) : typeof(int); @@ -915,8 +1133,31 @@ private static bool IsNumeric(Type type) return typeof(int); } + /// The type of -x or BNOT x. Keep in lock-step with ExpressionEvaluator.Negate and BitNot: + /// negation reads text as a Double and widens an Integer, Byte or Boolean to a Long; BNOT gives an Integer from + /// an Integer or a Boolean, an Int64 from an Int64, and otherwise a Long. + private static Type? DeclaredUnaryType(UnaryOperator op, Type? operand) + { + if (operand is null) + return null; + if (op == UnaryOperator.BitNot) + return IsSixteenBits(operand) ? typeof(short) : IsInt64(operand) ? typeof(long) : typeof(int); + if (operand == typeof(string)) + return typeof(double); + if (operand == typeof(short) || operand == typeof(byte) || operand == typeof(bool)) + return typeof(int); + return IsInt64(operand) ? typeof(long) : operand; + } + private static bool IsInt64(Type type) => type == typeof(long) || type == typeof(ulong); + /// A bitwise operand of 16 bits: an Integer or a Boolean (ExpressionEvaluator.BitOperand). + private static bool IsSixteenBits(Type? type) => type == typeof(short) || type == typeof(bool); + + private static bool IsSingleWidth(Type type) => type == typeof(float) || type == typeof(short) || type == typeof(bool); + + private static bool IsConcatText(Type type) => type == typeof(string) || type == typeof(Guid) || type == typeof(byte[]); + /// The set of source-table qualifiers that supply the DISTINCTROW projection's output columns. /// An unqualified column is resolved to its source table via the input's columns. private static HashSet ContributingQualifiers( @@ -940,11 +1181,7 @@ private static HashSet ContributingQualifiers( { ColumnReference c => [(c.Table, c.Column)], QualifiedStarExpression qs => [(qs.Table, "*")], - BinaryExpression b => ColumnRefs(b.Left).Concat(ColumnRefs(b.Right)), - UnaryExpression u => ColumnRefs(u.Operand), - FunctionCall f => f.Arguments.SelectMany(ColumnRefs), - InListExpression il => ColumnRefs(il.Value).Concat(il.Items.SelectMany(ColumnRefs)), - _ => [], + _ => expression.Operands()?.SelectMany(ColumnRefs) ?? [], }; /// @@ -1011,8 +1248,7 @@ private static HashSet ContributingQualifiers( var innerTable = _database.OpenTable(seek.Table); string innerAlias = seek.Alias ?? seek.Table; int innerWidth = innerTable.Definition.Columns.Count; - var seekColumns = innerTable.Definition.Columns - .Select(c => new OutputColumn(innerAlias, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(); + var seekColumns = innerTable.Definition.Columns.Select(c => OutputColumn.Of(innerAlias, c)).ToList(); var joinColumns = leftColumns.Concat(seekColumns).ToList(); int[] keyCols = seek.Index.Columns.Select(c => c.Column.Index).ToArray(); @@ -1402,12 +1638,7 @@ private ExpressionEvaluator Eval(IReadOnlyList columns, object?[] private (IReadOnlyList Columns, IEnumerable Rows) ExecuteWindow(WindowNode node, EvalScope? outer) { var (inColumns, inRowsEnum) = Execute(node.Input, outer); - - var columns = inColumns.Concat(node.Outputs.Select(o => new OutputColumn( - null, - o.Name, - WindowFunctions.Lookup(o.Function.Name).ResultType( - o.Function.Arguments.Count > 0 ? DeclaredType(o.Function.Arguments[0], inColumns) : null)))).ToList(); + var (types, columns) = WindowColumns(node.Outputs, inColumns); IEnumerable Rows() { @@ -1416,8 +1647,18 @@ private ExpressionEvaluator Eval(IReadOnlyList columns, object?[] for (int i = 0; i < rows.Count; i++) values[i] = new object?[node.Outputs.Count]; + // One scope/evaluator rebound per row, as the joins do: partition keys, sort keys and arguments are all + // evaluated once per row and a fresh pair each time is the dominant cost otherwise. + var scope = new EvalScope(inColumns, [], outer); + var eval = new ExpressionEvaluator(scope, this, parameters: _parameters, session: _session); for (int slot = 0; slot < node.Outputs.Count; slot++) - ComputeWindow(node.Outputs[slot].Function, rows, inColumns, outer, values, slot); + { + ComputeWindow(node.Outputs[slot].Function, rows.Count, i => + { + scope.Rebind(rows[i]); + return eval; + }, inColumns, values, slot, types[slot]); + } for (int i = 0; i < rows.Count; i++) yield return [.. rows[i], .. values[i]]; @@ -1426,29 +1667,71 @@ private ExpressionEvaluator Eval(IReadOnlyList columns, object?[] return (columns, Rows()); } - /// Computes one window function into of every row's value array. + /// The declared type of each window's column, and with those columns + /// appended. A windowed aggregate's column is typed as the grouped aggregate's, Currency and places included. + private (List Types, List Columns) WindowColumns( + IReadOnlyList windows, IReadOnlyList inColumns) + { + var types = windows + .Select(o => WindowFunctions.Lookup(o.Function.Name).ResultType(new WindowTyping(this, o.Function.Arguments, inColumns))) + .ToList(); + var columns = inColumns.Concat(windows.Select((o, i) => OutputColumn.Computed( + o.Name, + types[i], + QueryPlanner.IsAggregate(o.Function.Name) + ? ExpressionEvaluator.NumberTypeOf( + new FunctionCall(o.Function.Name, o.Function.Arguments, WithinGroup: o.Function.WithinGroup), + inColumns, e => DeclaredType(e, inColumns)) + : default, + o.Function))).ToList(); + return (types, columns); + } + + /// A window function's view of its arguments' declared types. + private sealed class WindowTyping(QueryExecutor executor, IReadOnlyList arguments, IReadOnlyList columns) + : IWindowTyping + { + public Type? ArgumentType(int argument) => + argument < arguments.Count ? executor.DeclaredType(arguments[argument], columns) : null; + + public Type? SharedType(params int[] indexes) + { + var present = indexes.Where(i => i < arguments.Count).Select(i => arguments[i]).ToList(); + return present.Any(a => a is not LiteralExpression { Value: null } && executor.DeclaredType(a, columns) is null) + ? null + : executor.UnifiedType(present, columns); + } + } + + /// Computes one window function into of every row's value array, each value as + /// the type the column has. The rows are input rows or, over + /// a grouped query, groups; gives the evaluator that sees the one at an index, and + /// is their schema. private void ComputeWindow( - WindowFunction fn, List rows, IReadOnlyList columns, EvalScope? outer, - object?[][] values, int slot) + WindowFunction fn, int count, Func evaluatorFor, IReadOnlyList columns, + object?[][] values, int slot, Type? declared) { WindowFunctionDef def = WindowFunctions.Lookup(fn.Name); if (fn.Arguments.Count < def.MinArguments || fn.Arguments.Count > def.MaxArguments) throw new InvalidOperationException( $"{fn.Name} takes {(def.MinArguments == def.MaxArguments ? $"{def.MinArguments}" : $"{def.MinArguments} to {def.MaxArguments}")} argument(s)."); - - // One scope/evaluator rebound per row, as the joins do: partition keys, sort keys and arguments are all - // evaluated once per row and a fresh pair each time is the dominant cost otherwise. - var scope = new EvalScope(columns, [], outer); - var eval = new ExpressionEvaluator(scope, this, parameters: _parameters, session: _session); + CheckOptions(fn, def); + WindowFrame? frame = fn.Over.Frame; + if (frame is not null) + CheckFrame(fn, frame); // Partition, preserving input order within each. A null partition key groups with other nulls exactly as // GROUP BY does, because this is the same key type. var partitions = new Dictionary>(); - var sortKeys = new object?[rows.Count][]; - var arguments = new object?[rows.Count][]; - for (int i = 0; i < rows.Count; i++) + var sortKeys = new object?[count][]; + var arguments = new object?[count][]; + // A frame's offsets, as each row evaluates them; the standard makes them constants, which is a special case. + var startOffsets = frame?.Start.Offset is null ? null : new object?[count]; + var endOffsets = frame?.End.Offset is null ? null : new object?[count]; + var included = fn.Filter is null ? null : new bool[count]; + for (int i = 0; i < count; i++) { - scope.Rebind(rows[i]); + ExpressionEvaluator eval = evaluatorFor(i); var key = new object?[fn.Over.PartitionBy.Count]; for (int k = 0; k < key.Length; k++) key[k] = eval.Evaluate(fn.Over.PartitionBy[k]); @@ -1464,9 +1747,24 @@ private void ComputeWindow( arguments[i] = fn.Arguments.Count == 0 ? [] : new object?[fn.Arguments.Count]; for (int k = 0; k < fn.Arguments.Count; k++) - arguments[i][k] = eval.Evaluate(fn.Arguments[k]); + arguments[i][k] = fn.Arguments[k] is StarExpression ? null : eval.Evaluate(fn.Arguments[k]); + if (startOffsets is not null) + startOffsets[i] = eval.Evaluate(frame!.Start.Offset!); + if (endOffsets is not null) + endOffsets[i] = eval.Evaluate(frame!.End.Offset!); + if (included is not null) + included[i] = eval.IsTrue(fn.Filter!); } + bool star = fn.Arguments is [StarExpression]; + var call = new WindowCall( + Star: star, + Currency: fn.Arguments is [var first] && !star && IsCurrency(first, columns), + Distinct: fn.Distinct, + IgnoreNulls: fn.IgnoreNulls == true, + FromLast: fn.FromLast == true, + WithinGroup: fn.WithinGroup); + foreach (List members in partitions.Values) { // Ties break on the original position, so the window order is stable and a window with no ORDER BY @@ -1489,25 +1787,71 @@ private void ComputeWindow( peerOrdinal[i] = samePeer ? peerOrdinal[i - 1] : peerOrdinal[i - 1] + 1; } + WindowFrameInput? frameInput = frame is null ? null : new WindowFrameInput( + frame, + startOffsets is null ? [] : members.Select(m => startOffsets[m]).ToList(), + endOffsets is null ? [] : members.Select(m => endOffsets[m]).ToList(), + frame.Unit == FrameUnit.Range && fn.Over.OrderBy.Count == 1 ? members.Select(m => sortKeys[m][0]).ToList() : null, + fn.Over.OrderBy is [{ Direction: SortDirection.Descending }]); + var output = new object?[members.Count]; def.Evaluate( - new WindowPartition(peerStart, peerOrdinal, members.Select(m => arguments[m]).ToList()), output); + new WindowPartition(peerStart, peerOrdinal, members.Select(m => arguments[m]).ToList(), call, frameInput, + included is null ? null : members.Select(m => included[m]).ToList()), + output); // Scatter back to the input positions: the node emits rows in input order, not window order. for (int i = 0; i < members.Count; i++) - values[members[i]][slot] = output[i]; + values[members[i]][slot] = ExpressionEvaluator.AsColumnType(output[i], declared, currency: false); } } + /// What the call is written with has to be what the function takes (): a frame + /// clause, RESPECT/IGNORE NULLS, FROM FIRST/LAST, DISTINCT. + private static void CheckOptions(WindowFunction fn, WindowFunctionDef def) + { + void Require(bool written, WindowOptions option, string what) + { + if (written && !def.Options.HasFlag(option)) + throw new InvalidOperationException($"{fn.Name} takes no {what}."); + } + Require(fn.Over.Frame is not null, WindowOptions.Frame, "window frame"); + Require(fn.IgnoreNulls is not null, WindowOptions.NullTreatment, "RESPECT NULLS or IGNORE NULLS"); + Require(fn.FromLast is not null, WindowOptions.FromLast, "FROM FIRST or FROM LAST"); + Require(fn.Distinct, WindowOptions.Distinct, "DISTINCT"); + Require(fn.Filter is not null, WindowOptions.Filter, "FILTER"); + } + + /// + /// The standard's rules for a frame clause that the window, not the syntax, decides: GROUPS counts peer groups, + /// so needs an ORDER BY; and a RANGE offset measures from one ORDER BY key, so needs exactly one. + /// + private static void CheckFrame(WindowFunction fn, WindowFrame frame) + { + if (frame.Unit == FrameUnit.Groups && fn.Over.OrderBy.Count == 0) + throw new InvalidOperationException("A GROUPS frame needs an ORDER BY in its window."); + if (frame.Unit == FrameUnit.Range && (frame.Start.Offset ?? frame.End.Offset) is not null && fn.Over.OrderBy.Count != 1) + throw new InvalidOperationException("A RANGE frame with an offset needs exactly one ORDER BY key in its window."); + } + private (IReadOnlyList Columns, IEnumerable Rows) ExecuteAggregate(AggregateNode node, EvalScope? outer) { var (inColumns, inRowsEnum) = Execute(node.Input, outer); + // Windows over the groups publish a column each, which the projection and ORDER BY read as any other. + IReadOnlyList windows = node.Windows ?? []; + var (windowTypes, columns) = WindowColumns(windows, inColumns); + + var outTypes = node.Projection + .Select(item => ExpressionEvaluator.NumberTypeOf(item.Value, columns, e => DeclaredType(e, columns))).ToList(); var outColumns = node.Projection - .Select((item, i) => new OutputColumn(null, + .Select((item, i) => OutputColumn.Computed( item.Alias ?? (item.Value is ColumnReference c ? c.Column : $"Expr{i + 1}"), - DeclaredType(item.Value, inColumns))) + DeclaredType(item.Value, columns), + outTypes[i], + item.Value)) .ToList(); + var conversions = node.Projection.Select(item => ChoiceConversion(item.Value, columns)).ToList(); // A bare `SELECT COUNT(*)` wants the number of rows, not the rows. Everything below materialises the // whole input first — which for this shape is the entire cost, and pure waste: holding every decoded row @@ -1519,16 +1863,15 @@ private void ComputeWindow( var inRows = inRowsEnum.ToList(); // Aggregates can appear in the projection, HAVING (e.g. HAVING COUNT(*) > 30) and ORDER BY // (e.g. ORDER BY COUNT(*)); precompute all of them per group so each instance resolves. + // A window's arguments and keys may hold aggregates too — RANK() OVER (ORDER BY SUM(x)). var aggregateCalls = node.Projection.SelectMany(i => Aggregates(i.Value)) .Concat(node.Having is { } h ? Aggregates(h) : []) .Concat(node.OrderBy.SelectMany(k => Aggregates(k.Value))) + .Concat(windows.SelectMany(w => w.Function.Expressions().SelectMany(Aggregates))) .ToList(); - // Each output row carries its ORDER BY key values AND its grouping-key values, evaluated in the same - // group scope as the projection, to sort the groups afterward: by ORDER BY if present, otherwise — - // matching Access, which returns GROUP BY results ascending by the grouping columns — by the group key - // (this also makes a TOP-1-over-a-GROUP-BY deterministic, as Access/SQL Server do). - var outRows = new List<(object?[] Row, object?[] SortKeys, object?[] GroupKeys)>(); + // The groups HAVING keeps, each with the row that resolves its keys and its aggregates' values. + var groups = new List<(object?[] KeyRow, Dictionary Values, ExpressionEvaluator Eval)>(); foreach (List group in GroupRows(inRows, node.GroupBy, inColumns, outer)) { var values = new Dictionary(ReferenceComparer.Instance); @@ -1551,8 +1894,32 @@ private void ComputeWindow( // HAVING filters whole groups after aggregation. if (node.Having is not null && !eval.IsTrue(node.Having)) continue; + groups.Add((keyRow, values, eval)); + } + + // The windows see the groups as their rows, as the standard orders it: after HAVING, before the projection. + var windowValues = new object?[groups.Count][]; + for (int i = 0; i < groups.Count; i++) + windowValues[i] = new object?[windows.Count]; + for (int slot = 0; slot < windows.Count; slot++) + ComputeWindow(windows[slot].Function, groups.Count, i => groups[i].Eval, inColumns, windowValues, slot, windowTypes[slot]); - object?[] row = node.Projection.Select(item => eval.Evaluate(item.Value)).ToArray(); + // Each output row carries its ORDER BY key values AND its grouping-key values, evaluated in the same + // group scope as the projection, to sort the groups afterward: by ORDER BY if present, otherwise — + // matching Access, which returns GROUP BY results ascending by the grouping columns — by the group key + // (this also makes a TOP-1-over-a-GROUP-BY deterministic, as Access/SQL Server do). + var outRows = new List<(object?[] Row, object?[] SortKeys, object?[] GroupKeys)>(); + for (int g = 0; g < groups.Count; g++) + { + var (keyRow, values, eval) = groups[g]; + if (windows.Count > 0) + eval = new ExpressionEvaluator( + new EvalScope(columns, [.. keyRow, .. windowValues[g]], outer, values), this, _parameters, _session); + + object?[] row = node.Projection + .Select((item, i) => ExpressionEvaluator.ToResultPlaces( + ExpressionEvaluator.AsColumnType(eval.Evaluate(item.Value), conversions[i], currency: false), outTypes[i])) + .ToArray(); object?[] sortKeys = node.OrderBy.Select(k => eval.Evaluate(k.Value)).ToArray(); object?[] groupKeys = node.GroupBy.Select(k => eval.Evaluate(k)).ToArray(); outRows.Add((row, sortKeys, groupKeys)); @@ -1591,7 +1958,7 @@ private static bool IsBareCountStar(AggregateNode node) => node.GroupBy.Count == 0 && node.Having is null && node.OrderBy.Count == 0 - && node.Projection is [{ Value: FunctionCall { Distinct: false, Arguments: [StarExpression] } call }] + && node.Projection is [{ Value: FunctionCall { Distinct: false, Filter: null, Arguments: [StarExpression] } call }] && string.Equals(call.Name, "COUNT", StringComparison.OrdinalIgnoreCase); /// Counts a row sequence without retaining it. COUNT is an Access Long Integer, so the count is an @@ -1643,14 +2010,13 @@ private static int CountRows(IEnumerable rows) ExpressionEvaluator.ValidateArity(name, call.Arguments.Count); Expression? arg = call.Arguments.Count > 0 ? call.Arguments[0] : null; - // COUNT is an Access Long Integer (32-bit) — EF reads it with GetInt32, so return int, not long. - if (name == "COUNT") - { - if (arg is StarExpression or null) - return group.Count; // COUNT(*) counts rows; DISTINCT is meaningless (and EF never emits it) - var counted = group.Select(r => Eval(columns, r, outer).Evaluate(arg)).Where(v => v is not null); - return call.Distinct ? DistinctValues(counted).Count : counted.Count(); - } + // FILTER (WHERE …) narrows the group before anything else looks at it — COUNT(*) included. + if (call.Filter is { } filter) + group = group.Where(r => Eval(columns, r, outer).IsTrue(filter)).ToList(); + + // COUNT(*) counts rows; DISTINCT is meaningless there (and EF never emits it). + if (name == "COUNT" && arg is StarExpression or null) + return group.Count; // FIRST/LAST return the argument's value from the first/last row of the group in scan order — NOT // null-filtered (verified vs ACE: First over a leading NULL row returns NULL). @@ -1659,59 +2025,59 @@ private static int CountRows(IEnumerable rows) if (name == "LAST") return group.Count == 0 ? null : Eval(columns, group[^1], outer).Evaluate(arg!); - var values = group.Select(r => Eval(columns, r, outer).Evaluate(arg!)).Where(v => v is not null).ToList(); - // SUM(DISTINCT)/AVG(DISTINCT)/… aggregate the distinct set of the argument's values. MIN/MAX are + if (call.WithinGroup is { } directions) + { + if (group.Count == 0) + return null; + if (name == "LISTAGG") + { + IReadOnlyList keys = call.WithinGroupKeys; + return ListAgg.Of( + group.Select(r => + { + ExpressionEvaluator e = Eval(columns, r, outer); + return (e.Evaluate(call.Arguments[0]), keys.Select(k => e.Evaluate(k)).ToArray()); + }), + call.Arguments.Count - keys.Count == 2 ? (string)((LiteralExpression)call.Arguments[1]).Value! : "", + directions, + call.Distinct); + } + // The fraction is the group's, so any row gives it; the standard makes it a constant. + return Percentile.Of(name, + group.Select(r => Eval(columns, r, outer).Evaluate(call.Arguments[1])), + Eval(columns, group[0], outer).Evaluate(call.Arguments[0]), + directions[0]); + } + + // A binary set function reads a pair from each row; the standard gives it no DISTINCT. + if (RunningAggregate.IsPair(name)) + { + if (call.Distinct) + throw new NotSupportedException($"{call.Name} takes no DISTINCT."); + var pair = new RunningAggregate(name, countRows: false, currency: false); + foreach (object?[] row in group) + { + ExpressionEvaluator rowEval = Eval(columns, row, outer); + pair.AddPair(rowEval.Evaluate(call.Arguments[0]), rowEval.Evaluate(call.Arguments[1])); + } + return pair.Result; + } + + IEnumerable values = group.Select(r => Eval(columns, r, outer).Evaluate(arg!)); + // COUNT(DISTINCT)/SUM(DISTINCT)/… aggregate the distinct set of the argument's values. MIN/MAX are // unaffected by dedup, but applying it uniformly keeps the one code path. if (call.Distinct) - values = DistinctValues(values); - if (values.Count == 0) - return null; // SUM/AVG/MIN/MAX of nothing is NULL (COUNT already returned above) - - // Result types: SUM **preserves the input type** (int→int, long→long, decimal→decimal, …) so the EF - // provider (which emits a bare SUM and reads by the LINQ operand type) round-trips without a cast. - // AVG is Double unless the input is Currency/Decimal (matches Access and LINQ). MIN/MAX keep the - // column's own value and type. - var inv = System.Globalization.CultureInfo.InvariantCulture; - return name switch - { - "SUM" => SumPreservingType(values, inv), - "AVG" => values[0] is decimal ? values.Average(v => Convert.ToDecimal(v, inv)) : values.Average(v => Convert.ToDouble(v, inv)), - "MIN" => values.Aggregate((a, b) => ExpressionEvaluator.CompareForSort(a, b) <= 0 ? a : b), - "MAX" => values.Aggregate((a, b) => ExpressionEvaluator.CompareForSort(a, b) >= 0 ? a : b), - // Statistical aggregates. Sample forms (StDev/Var) divide by n-1 and are NULL for a single value; - // population forms (StDevP/VarP) divide by n. Verified vs ACE. - "VAR" or "STDEV" or "STDDEV" or "VARP" or "STDEVP" or "STDDEVP" => Statistic(name, values, inv), - _ => throw new NotSupportedException($"Aggregate {call.Name} is not supported."), - }; - } + values = DistinctValues(values.Where(v => v is not null)); - /// Access statistical aggregates over the non-null values (verified vs ACE). VAR/STDEV are the - /// **sample** forms (divide by n−1, NULL for a single value); VARP/STDEVP the **population** forms (divide by - /// n). STDEV/STDEVP are the square roots of VAR/VARP. - private static object? Statistic(string name, List values, System.Globalization.CultureInfo inv) - { - int n = values.Count; - double mean = values.Average(v => Convert.ToDouble(v, inv)); - double sumSq = values.Sum(v => { double d = Convert.ToDouble(v, inv) - mean; return d * d; }); - bool sample = !name.EndsWith("P", StringComparison.Ordinal); // VAR/STDEV sample; VARP/STDEVP population - if (sample && n < 2) return null; // sample variance of one value is undefined - double variance = sumSq / (sample ? n - 1 : n); - bool stdev = name.Contains("DEV", StringComparison.Ordinal); - return stdev ? Math.Sqrt(variance) : variance; + var aggregate = new RunningAggregate(name, countRows: false, currency: IsCurrency(arg!, columns)); + foreach (object? value in values) + aggregate.Add(value); + return aggregate.Result; } - /// SUM keeping the operand's numeric type (as LINQ's Sum overloads do): integer types - /// (byte/short/int) sum to Int32, Int64 to Int64, Single to Single, Double to Double, Decimal/Currency - /// to Decimal. - private static object SumPreservingType(List values, System.Globalization.CultureInfo inv) => - values[0] switch - { - decimal => values.Sum(v => Convert.ToDecimal(v, inv)), - double => values.Sum(v => Convert.ToDouble(v, inv)), - float => (float)values.Sum(v => Convert.ToDouble(v, inv)), - long or ulong => values.Sum(v => Convert.ToInt64(v, inv)), - _ => values.Sum(v => Convert.ToInt32(v, inv)), - }; + /// Whether an aggregate's argument is a Currency, which the statistical aggregates square exactly. + private bool IsCurrency(Expression argument, IReadOnlyList columns) => + ExpressionEvaluator.NumberTypeOf(argument, columns, e => DeclaredType(e, columns)).Class == NumberClass.Currency; private static IEnumerable Aggregates(Expression e) { @@ -1720,15 +2086,6 @@ private static IEnumerable Aggregates(Expression e) case FunctionCall f when QueryPlanner.IsAggregate(f.Name): yield return f; break; - case FunctionCall f: - foreach (FunctionCall a in f.Arguments.SelectMany(Aggregates)) yield return a; - break; - case BinaryExpression b: - foreach (FunctionCall a in Aggregates(b.Left).Concat(Aggregates(b.Right))) yield return a; - break; - case UnaryExpression u: - foreach (FunctionCall a in Aggregates(u.Operand)) yield return a; - break; // Descend into subqueries: an aggregate over an *outer* column may appear there (a correlated // subquery). Its own aggregates come along too, but are skipped when they can't be computed in // this group's scope. @@ -1741,17 +2098,12 @@ private static IEnumerable Aggregates(Expression e) case InSubqueryExpression i: foreach (FunctionCall a in Aggregates(i.Value).Concat(AggregatesInSelect(i.Query))) yield return a; break; - case InListExpression i: - foreach (FunctionCall a in Aggregates(i.Value).Concat(i.Items.SelectMany(Aggregates))) yield return a; - break; - // Both halves of every arm, and the ELSE. An aggregate in a CASE is computed for the group up - // front and handed to the evaluator by reference — the standard specifies the same order, that - // aggregates in a WHEN are evaluated before the CASE rather than by it. Conditions matter as much + // Operands include both halves of every CASE arm, and the ELSE. An aggregate in a CASE is computed for + // the group up front and handed to the evaluator by reference — the standard specifies the same order, + // that aggregates in a WHEN are evaluated before the CASE rather than by it. Conditions matter as much // as results: `HAVING CASE WHEN COUNT(*) > 1 THEN …` carries the aggregate in the condition. - case CaseExpression c: - foreach (FunctionCall a in c.WhenClauses - .SelectMany(w => Aggregates(w.Condition).Concat(Aggregates(w.Result))) - .Concat(c.ElseResult is { } e2 ? Aggregates(e2) : [])) + default: + foreach (FunctionCall a in e.Operands()?.SelectMany(Aggregates) ?? []) yield return a; break; } @@ -1778,7 +2130,7 @@ private static IEnumerable Aggregates(Expression e) /// Groups by structural equality of the key value tuple. // DISTINCT / GROUP BY / INTERSECT / EXCEPT key. String keys use Access text semantics — case-insensitive // and trailing-space-insensitive — so 'London' and 'LONDON ' group together as Access does. - private sealed class GroupKey(object?[] values) : IEquatable + internal sealed class GroupKey(object?[] values) : IEquatable { private readonly object?[] _values = values; diff --git a/src/LibRed/LibRed.Engine/Execution/RunningAggregate.cs b/src/LibRed/LibRed.Engine/Execution/RunningAggregate.cs new file mode 100644 index 000000000..faa3e16e6 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/RunningAggregate.cs @@ -0,0 +1,237 @@ +using System.Globalization; +using EntityFrameworkCore.Jet.Data; + +namespace LibRed.Engine.Execution; + +/// +/// An aggregate fed its argument's values one at a time, whose result can be read after any of them — the arithmetic +/// behind both GROUP BY's aggregates and the windowed ones, so the two give the same values. Null values are skipped, +/// except where COUNT(*) counts rows. +/// +/// +/// COUNT is an Access Long Integer (32-bit) — EF reads it with GetInt32, so it is an int, not a long. +/// The numeric aggregates read each value as the conversion functions read it (verified vs ACE): text as a +/// number (text that is not one is a type mismatch), a date as its serial, True as -1; a GUID or binary value is a +/// type mismatch. SUM keeps the type of the first value, as LINQ's Sum overloads do — the EF provider emits a +/// bare SUM and reads it by the LINQ operand type: Boolean, Byte, Integer and Long sum to Int32, Int64 to Int64, Single +/// to Single, Double (text and dates included) to Double, Decimal and Currency to Decimal. AVG is Double unless the +/// first value is a Decimal or Currency, and a Decimal average keeps its full precision, as LINQ's does; ACE rounds a +/// Currency average to four places and cuts a Decimal one to ten. +/// MIN and MAX keep the value and its type, in the sort order. Empty text wins against any other text but then +/// counts as no value, so the next value replaces it, and empty text left at the end is Null (verified vs ACE: Min of +/// '5', 'x', '', '7' is '7', and Min of '5', '' is Null). +/// The statistical aggregates are verified vs ACE to the last bit. VAR and STDEV are the sample forms, Null for +/// a single value; VARP and STDEVP the population forms; the STDEVs are the square roots. ACE works them out as +/// (n·Σx² − (Σx)²) / (n·(n−1)), or / n² for the population, in doubles; a Single is squared in single precision, and +/// a Currency's square and squared sum are Currency products, rounded to four places. The standard's names — +/// STDDEV_SAMP, STDDEV_POP, VAR_SAMP and VAR_POP — are the same aggregates. +/// The standard's binary set functions — CORR, COVAR_POP, COVAR_SAMP and the REGR_ family — take a pair, the +/// dependent value first (), and use only the pairs where neither is Null. ACE has none of them, +/// so there is nothing to match bit for bit: the sums of squares and products are kept by the Youngs–Cramer update, +/// as PostgreSQL keeps them, which does not lose the small differences that the textbook Σx² − (Σx)²/n cancels away. +/// REGR_COUNT is an int, as COUNT is; the rest are Doubles, Null over no pairs. The standard's rules for a degenerate +/// input apply: COVAR_SAMP needs two pairs; REGR_SLOPE, REGR_INTERCEPT and REGR_R2 are Null when every x is the same, +/// and CORR when every x or every y is; REGR_R2 is 1 when every y is the same. +/// +internal sealed class RunningAggregate +{ + private static readonly Dictionary StandardNames = new() + { + ["STDDEV_SAMP"] = "STDEV", + ["STDDEV_POP"] = "STDEVP", + ["VAR_SAMP"] = "VAR", + ["VAR_POP"] = "VARP", + }; + + private static readonly HashSet PairNames = + [ + "CORR", "COVAR_POP", "COVAR_SAMP", "REGR_COUNT", "REGR_AVGX", "REGR_AVGY", "REGR_SXX", "REGR_SYY", "REGR_SXY", + "REGR_SLOPE", "REGR_INTERCEPT", "REGR_R2", + ]; + + private readonly string _name; + private readonly bool _countRows; + private readonly bool _currency; + + // The pair aggregates: sums of x and y, and the sums of squared and multiplied deviations from their means. + private double _sumX, _sumY, _sxx, _syy, _sxy; + + private int _count; + private object? _extreme; + private object? _first; + private decimal _decimalSum; + private double _doubleSum; + private long _longSum; + private int _intSum; + private double _squares; + private decimal _exactSum; + private decimal _exactSquares; + private bool _allDecimal = true; + + /// The aggregate, in upper case. + /// Whether this is COUNT(*), which counts rows rather than values. + /// Whether the argument is a Currency, which the statistical aggregates square + /// exactly. + public RunningAggregate(string name, bool countRows, bool currency) + { + _name = Supports(name) ? Canonical(name) : throw new NotSupportedException($"Aggregate {name} is not supported."); + _countRows = countRows; + _currency = currency; + } + + /// Whether (upper case) is an aggregate this computes. + public static bool Supports(string name) => + Canonical(name) is "COUNT" or "SUM" or "AVG" or "MIN" or "MAX" or "VAR" or "VARP" or "STDEV" or "STDEVP" + or "STDDEV" or "STDDEVP" + || IsPair(name); + + /// Whether (upper case) is a binary set function, fed by . + public static bool IsPair(string name) => PairNames.Contains(name); + + /// The Access name for one of the standard's: STDDEV_SAMP is STDEV, VAR_POP is VARP, …. + public static string Canonical(string name) => StandardNames.GetValueOrDefault(name, name); + + /// Adds a pair to a binary set function: , the dependent value, and + /// . The pair counts only when neither is Null; each is read as the conversion functions + /// read a value. + public void AddPair(object? y, object? x) + { + if (y is null || x is null) + return; + double dy = Convert.ToDouble(ExpressionEvaluator.ConversionNumber(y), CultureInfo.InvariantCulture); + double dx = Convert.ToDouble(ExpressionEvaluator.ConversionNumber(x), CultureInfo.InvariantCulture); + double n = ++_count; + _sumX += dx; + _sumY += dy; + if (n > 1) + { + double tx = dx * n - _sumX, ty = dy * n - _sumY, scale = 1.0 / (n * (n - 1)); + _sxx += tx * tx * scale; + _syy += ty * ty * scale; + _sxy += tx * ty * scale; + } + } + + private static bool IsStatistic(string name) => name is not ("COUNT" or "SUM" or "AVG" or "MIN" or "MAX"); + + public void Add(object? value) + { + if (_countRows) + { + _count++; + return; + } + if (value is null) + return; + _count++; + + if (_name is "COUNT") + return; + if (_name is "MIN" or "MAX") + { + if (_extreme is null or string { Length: 0 } + || (_name == "MAX" + ? ExpressionEvaluator.CompareForSort(value, _extreme) > 0 + : ExpressionEvaluator.CompareForSort(value, _extreme) < 0)) + _extreme = value; + return; + } + + object number = ExpressionEvaluator.ConversionNumber(value); + _first ??= number; + CultureInfo invariant = CultureInfo.InvariantCulture; + if (IsStatistic(_name)) + { + if (number is not decimal) + _allDecimal = false; + else if (_currency) + { + decimal m = (decimal)number; + _exactSum += m; + _exactSquares += decimal.Round(m * m, 4); + } + + if (number is float f) + { + _doubleSum += f; + _squares += f * f; + } + else + { + double d = Convert.ToDouble(number, invariant); + _doubleSum += d; + _squares += d * d; + } + return; + } + + switch (_first) + { + case decimal: + _decimalSum += JetDecimalConverter.ToDecimal(number, invariant); + break; + case long or ulong when _name == "SUM": + _longSum = checked(_longSum + Convert.ToInt64(number, invariant)); + break; + case double or float: + case not null when _name == "AVG": + _doubleSum += Convert.ToDouble(number, invariant); + break; + default: + _intSum = checked(_intSum + Convert.ToInt32(number, invariant)); + break; + } + } + + /// The aggregate of the values added so far. + public object? Result => _name switch + { + "COUNT" or "REGR_COUNT" => _count, + "MIN" or "MAX" => _extreme is string { Length: 0 } ? null : _extreme, + _ when _count == 0 => null, + _ when IsPair(_name) => PairResult(), + "SUM" => _first switch + { + decimal => _decimalSum, + double => _doubleSum, + float => (float)_doubleSum, + long or ulong => _longSum, + _ => _intSum, + }, + "AVG" => _first is decimal ? _decimalSum / _count : _doubleSum / _count, + _ => Statistic(), + }; + + private double? PairResult() + { + double n = _count; + return _name switch + { + "COVAR_POP" => _sxy / n, + "COVAR_SAMP" => n < 2 ? null : _sxy / (n - 1), + "CORR" => _sxx == 0 || _syy == 0 ? null : _sxy / (Math.Sqrt(_sxx) * Math.Sqrt(_syy)), + "REGR_AVGX" => _sumX / n, + "REGR_AVGY" => _sumY / n, + "REGR_SXX" => _sxx, + "REGR_SYY" => _syy, + "REGR_SXY" => _sxy, + "REGR_SLOPE" => _sxx == 0 ? null : _sxy / _sxx, + "REGR_INTERCEPT" => _sxx == 0 ? null : (_sumY - _sumX * _sxy / _sxx) / n, + _ => _sxx == 0 ? null : _syy == 0 ? 1.0 : _sxy * _sxy / (_sxx * _syy), // REGR_R2 + }; + } + + private object? Statistic() + { + bool sample = !_name.EndsWith('P'); + if (sample && _count < 2) + return null; + + (double squares, double squaredSum) = _currency && _allDecimal + ? ((double)_exactSquares, (double)decimal.Round(_exactSum * _exactSum, 4)) + : (_squares, _doubleSum * _doubleSum); + double n = _count; + double variance = (n * squares - squaredSum) / (sample ? n * (n - 1) : n * n); + return _name.Contains("DEV", StringComparison.Ordinal) ? Math.Sqrt(variance) : variance; + } +} diff --git a/src/LibRed/LibRed.Engine/Execution/StatementExecutor.cs b/src/LibRed/LibRed.Engine/Execution/StatementExecutor.cs index 6c976e951..0bad79747 100644 --- a/src/LibRed/LibRed.Engine/Execution/StatementExecutor.cs +++ b/src/LibRed/LibRed.Engine/Execution/StatementExecutor.cs @@ -1,4 +1,5 @@ using LibRed.Catalog; +using LibRed.Engine.Planning; using LibRed.Sql.Ast; using LibRed.Sql.Parsing; using LibRed.Storage; @@ -66,16 +67,7 @@ private int ExecuteCreateTable(CreateTableStatement statement) ValidateColumnDefault(spec, def.Default); IReadOnlyList? primaryKey = statement.PrimaryKey.Count > 0 ? statement.PrimaryKey : null; - var relationships = statement.ForeignKeys.Select(fk => new RelationshipSpec( - Name: fk.Name ?? DefaultRelationshipName(statement.Table, fk), - ReferencedTable: fk.ReferencedTable, - Columns: PairColumns(fk), - IsEnforced: true, - CascadeUpdate: fk.OnUpdate == ReferentialAction.Cascade, - CascadeDelete: fk.OnDelete == ReferentialAction.Cascade, - NoIndex: fk.NoIndex, - DeleteSetNull: fk.OnDelete == ReferentialAction.SetNull, - UpdateSetNull: fk.OnUpdate == ReferentialAction.SetNull)).ToList(); + var relationships = statement.ForeignKeys.Select(fk => ToRelationshipSpec(statement.Table, fk)).ToList(); var uniques = statement.UniqueConstraints.Select((u, i) => new UniqueIndexSpec( Name: u.Name ?? $"UQ_{statement.Table}_{i}", @@ -128,8 +120,7 @@ private static void EnforceRequired(string table, IReadOnlyList colum private void EnforceCheckConstraints(TableDef definition, object?[] values) { if (definition.CheckConstraints.Count == 0) return; - var schema = definition.Columns - .Select(c => new OutputColumn(definition.Name, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(); + var schema = definition.Columns.Select(c => OutputColumn.Of(definition.Name, c)).ToList(); var evaluator = new ExpressionEvaluator(new EvalScope(schema, values, null), _scalarRunner, _parameters, _session); foreach (var (name, expression) in definition.CheckConstraints) if (evaluator.Evaluate(_parser.ParseExpression(expression)) is false) @@ -138,14 +129,22 @@ private void EnforceCheckConstraints(TableDef definition, object?[] values) "Enter a value that the expression for this field can accept."); } - /// Pairs each child FK column with its referenced parent column, in key order. - private static List<(string Column, string ReferencedColumn)> PairColumns(ForeignKeyConstraint fk) - { - if (fk.ReferencedColumns.Count != fk.Columns.Count) - throw new InvalidOperationException( - $"Foreign key on '{fk.ReferencedTable}' has {fk.Columns.Count} columns but references {fk.ReferencedColumns.Count}."); - return fk.Columns.Zip(fk.ReferencedColumns).ToList(); - } + /// The relationship a parsed foreign key creates on , for CREATE TABLE and + /// ALTER TABLE alike. A REFERENCES table with no column list names only its child columns; the table + /// creator resolves it to the parent's primary key. + private static RelationshipSpec ToRelationshipSpec(string childTable, ForeignKeyConstraint fk) => new( + Name: fk.Name ?? DefaultRelationshipName(childTable, fk), + ReferencedTable: fk.ReferencedTable, + Columns: fk.ReferencedColumns.Count == 0 + ? RelationshipSpec.ChildColumnsOnly(fk.Columns) + : RelationshipSpec.PairColumns(fk.ReferencedTable, fk.Columns, fk.ReferencedColumns), + IsEnforced: true, + CascadeUpdate: fk.OnUpdate == ReferentialAction.Cascade, + CascadeDelete: fk.OnDelete == ReferentialAction.Cascade, + NoIndex: fk.NoIndex, + DeleteSetNull: fk.OnDelete == ReferentialAction.SetNull, + UpdateSetNull: fk.OnUpdate == ReferentialAction.SetNull, + ReferencesPrimaryKey: fk.ReferencedColumns.Count == 0); /// Access-style fallback name when the constraint is unnamed: "childparent". private static string DefaultRelationshipName(string childTable, ForeignKeyConstraint fk) => @@ -448,7 +447,7 @@ private int ExecuteCreateProcedure(CreateProcedureStatement statement) // MSysRelationships, so removing those rows disables it. DropConstraintAction drop => DropConstraint(statement.Table, drop.Name), // ADD COLUMN: append the column's descriptor/name to the TDEF (existing rows read it as NULL). - AddColumnAction add => AddColumn(statement.Table, add.Column), + AddColumnAction add => AddColumn(statement.Table, add), // DROP COLUMN: a metadata-only TDEF edit (remove the descriptor + name, decrement ColumnCount). DropColumnAction dropCol => DropColumn(statement.Table, dropCol.Field), // ALTER COLUMN field type: change the column's declared type (a variable text/binary length change is a @@ -525,13 +524,23 @@ private int DropConstraint(string table, string name) "constraint of that name exists."); } - private int AddColumn(string table, ColumnDefinition column) + private int AddColumn(string table, AddColumnAction add) { + ColumnDefinition column = add.Column; // NOT NULL and DEFAULT are written to the column's LvProp properties (Required / DefaultValue). ColumnSpec spec = MapColumn(column); ValidateColumnDefault(spec, column.Default); if (!_database.AddColumn(table, spec, column.Default)) throw new InvalidOperationException($"ALTER TABLE '{table}' ADD COLUMN '{column.Name}': the column already exists."); + // The column's constraints apply to it as in CREATE TABLE — a primary key, a unique index, a relationship — + // in that order, as ACE's ADD COLUMN does. Each fails the whole statement when ACE's would: a second + // primary key, or a primary key over a column the existing rows leave NULL. + if (column.PrimaryKey) + AddPrimaryKey(table, new AddPrimaryKeyAction(add.PrimaryKeyName, [column.Name])); + if (add.Unique is { } unique) + AddUnique(table, new AddUniqueAction(unique)); + if (add.References is { } references) + AddForeignKey(table, references); return 0; } @@ -619,7 +628,8 @@ private int AddCheck(string table, AddCheckAction chk) private int AlterColumn(string table, AlterColumnAction alter) { - var colDef = new ColumnDefinition(alter.Field, alter.TypeName, alter.Size, alter.Scale, NotNull: false, PrimaryKey: false); + var colDef = new ColumnDefinition(alter.Field, alter.TypeName, alter.Size, alter.Scale, NotNull: false, PrimaryKey: false, + Identity: alter.Identity); ColumnSpec spec = MapColumn(colDef); // Validated against the column's NEW type, and before the type change, so a rejected DEFAULT leaves // the whole statement having done nothing. @@ -648,16 +658,7 @@ private int DropColumnDefault(string table, AlterColumnDropDefaultAction drop) private int AddForeignKey(string table, ForeignKeyConstraint fk) { - _database.AddForeignKey(table, new RelationshipSpec( - Name: fk.Name ?? DefaultRelationshipName(table, fk), - ReferencedTable: fk.ReferencedTable, - Columns: PairColumns(fk), - IsEnforced: true, - CascadeUpdate: fk.OnUpdate == ReferentialAction.Cascade, - CascadeDelete: fk.OnDelete == ReferentialAction.Cascade, - NoIndex: fk.NoIndex, - DeleteSetNull: fk.OnDelete == ReferentialAction.SetNull, - UpdateSetNull: fk.OnUpdate == ReferentialAction.SetNull)); + _database.AddForeignKey(table, ToRelationshipSpec(table, fk)); return 0; } @@ -796,6 +797,40 @@ private Dictionary SourceColumnsFor(SelectStatement statement /// from: NULL stores NULL, DEFAULT takes whatever the column declares. private static readonly object DefaultRowValue = new(); + /// A table's columns with a DEFAULT value (parsed once), and its AutoNumber column if it has one — Jet + /// allows at most one, and its value after an insert is @@IDENTITY. + private sealed record RowDefaults(List<(int Index, Expression Expression)> Columns, ColumnDef? AutoNumber); + + /// + /// The defaults a new row takes for the columns it omits, matching Access — EF Core relies on the store default + /// rather than supplying the value itself. AutoNumber columns are excluded: their value is assigned by the row + /// inserter (sequential counter, or a random Int32 for a GenUniqueID() "Random" AutoNumber), not by evaluating + /// the DefaultValue — and GenUniqueID() is not a callable expression, so parsing it as a default would fail. + /// + private RowDefaults DefaultsOf(TableDef definition) => new( + definition.Columns + .Where(c => c.DefaultValue is not null && !c.IsAutoNumber) + .Select(c => (c.Index, Expression: ParseDefaultExpression(c.DefaultValue!))) + .ToList(), + definition.Columns.FirstOrDefault(c => c.IsAutoNumber)); + + /// Writes a new row: every column not in takes its default (an explicit + /// NULL is left as NULL), the row is checked as any insert is, and it is written. Returns its AutoNumber value, + /// if it has one. + private object? InsertNewRow(string tableName, Table table, RowDefaults defaults, object?[] values, IReadOnlySet provided) + { + var evaluator = new ExpressionEvaluator(new EvalScope([], [], null), _scalarRunner, parameters: _parameters); + foreach (var (index, expression) in defaults.Columns) + if (!provided.Contains(index)) + values[index] = evaluator.Evaluate(expression); + + EnforceRequired(tableName, table.Definition.Columns, values); + EnforceReferentialIntegrity(tableName, table, values); + EnforceCheckConstraints(table.Definition, values); + table.Insert(values); // fills values[autoNumber.Index] with the generated id (array mutated in place) + return defaults.AutoNumber is { } autoNumber ? values[autoNumber.Index] : null; + } + private int ExecuteInsert(InsertStatement statement) { Table table = _database.OpenTable(statement.Table); @@ -811,19 +846,7 @@ private int ExecuteInsert(InsertStatement statement) var evaluator = new ExpressionEvaluator( new EvalScope([], [], null), _scalarRunner, parameters: _parameters); - - // Columns with a DEFAULT value (parsed once): applied to any row that omits the column, matching - // Access — EF Core relies on the store default rather than supplying the value itself. - // AutoNumber columns are excluded: their value is assigned by the row inserter (sequential counter, or - // a random Int32 for a GenUniqueID() "Random" AutoNumber), not by evaluating the DefaultValue — and - // GenUniqueID() is not a callable expression, so parsing it as a default would fail. - var defaultColumns = columns - .Where(c => c.DefaultValue is not null && !c.IsAutoNumber) - .Select(c => (c.Index, Expression: ParseDefaultExpression(c.DefaultValue!))) - .ToList(); - - // Jet allows at most one AutoNumber column; its post-insert value is @@IDENTITY. - ColumnDef? autoNumber = columns.FirstOrDefault(c => c.IsAutoNumber); + RowDefaults defaults = DefaultsOf(table.Definition); int affected = 0; object? lastIdentity = null; @@ -844,7 +867,7 @@ void InsertRow(ReadOnlySpan supplied) ?? throw new InvalidOperationException($"Column '{targets[i]}' does not exist in '{statement.Table}'."); // An explicit DEFAULT is not a value: leaving the column out of `provided` routes it through - // the default-filling loop below, so it takes its declared default, or stays NULL when it has + // InsertNewRow's default filling, so it takes its declared default, or stays NULL when it has // none — which is what the standard specifies. A NOT NULL column with no default then fails // EnforceRequired, as it should. An AutoNumber column takes its generated id, the same as it // would from INSERT INTO t DEFAULT VALUES. @@ -855,17 +878,9 @@ void InsertRow(ReadOnlySpan supplied) provided.Add(column.Index); } - // Fill defaults for columns the insert didn't mention (an explicit NULL is left as NULL). - foreach (var (index, expression) in defaultColumns) - if (!provided.Contains(index)) - values[index] = evaluator.Evaluate(expression); - - EnforceRequired(statement.Table, columns, values); - EnforceReferentialIntegrity(statement.Table, table, values); - EnforceCheckConstraints(table.Definition, values); - table.Insert(values); // fills values[autoNumber.Index] with the generated id (array mutated in place) - if (autoNumber is not null) - lastIdentity = values[autoNumber.Index]; + object? identity = InsertNewRow(statement.Table, table, defaults, values, provided); + if (defaults.AutoNumber is not null) + lastIdentity = identity; affected++; } @@ -915,7 +930,7 @@ void InsertRow(ReadOnlySpan supplied) if (_session is not null) { _session.RowCount = affected; - if (autoNumber is not null) + if (defaults.AutoNumber is not null) _session.LastIdentity = lastIdentity; } return affected; @@ -927,8 +942,19 @@ void InsertRow(ReadOnlySpan supplied) /// null and its already-materialised (never a target). /// The plan of a LATERAL source — an APPLY's right side — which is re-executed once /// per outer row instead of being materialised, because it may correlate to the rows joined before it. + /// A writable derived table is one source table per table it reads, in a run: the first carries the + /// joined rows the derived query chose (, tables wide) and the rest + /// are , filled from them. tells a table's rows apart from another + /// table's under the same alias. private sealed record SourceTable(string Alias, Table? Table, IReadOnlyList Columns, - IReadOnlyList? DerivedRows, Plan.PlanNode? Lateral = null); + IReadOnlyList? DerivedRows, Plan.PlanNode? Lateral = null) + { + public IReadOnlyList<(RowId Id, object?[] Values)[]>? Combos { get; init; } + public int ComboWidth { get; init; } = 1; + public bool InCombo { get; init; } + public string? Key { get; init; } + public string CacheKey => Key ?? Alias; + } /// How a join kind is handled in a DML source. private enum JoinShape @@ -966,30 +992,51 @@ private enum JoinShape /// Flattens the UPDATE/DELETE table source into its tables in order, each paired with the join that /// introduced it: its and ON condition (the first/base table is Inner with a null ON). - /// INNER/CROSS/LEFT joins over named tables are supported — the left-deep form EF and Access emit. - private (List Tables, List Kinds, List Ons) ResolveSource(TableReference from) + /// INNER/CROSS/LEFT joins over named tables are supported — the left-deep form EF and Access emit — and the + /// bracketed groups WalkGroup describes. A table's group base, when it has one, is the table it is null-extended + /// with. + private (List Tables, List Kinds, List Ons, List GroupBases) ResolveSource( + TableReference from) { var tables = new List(); var kinds = new List(); var ons = new List(); + var groupBases = new List(); void EmitTable(NamedTable n, JoinKind kind, Expression? on) { Table t = _database.OpenTable(n.Name); string alias = n.Alias ?? n.Name; - tables.Add(new SourceTable(alias, t, t.Definition.Columns - .Select(c => new OutputColumn(alias, c.Name, Schema.JetClrTypeMap.ToClrType(c.Type))).ToList(), null)); + tables.Add(new SourceTable(alias, t, t.Definition.Columns.Select(c => OutputColumn.Of(alias, c)).ToList(), null)); kinds.Add(kind); ons.Add(on); + groupBases.Add(null); } void EmitDerived(SubqueryTable sq, JoinKind kind, Expression? on) { - string alias = sq.Alias ?? throw new NotSupportedException("A derived table in an UPDATE/DELETE source requires an alias."); + if (WritableDerivedTable(sq) is { } writable) + { + // The ON and join kind go with the run's first table, which brings in all of the run's rows. + foreach (SourceTable table in writable) + { + tables.Add(table); + kinds.Add(table.InCombo ? JoinKind.Inner : kind); + ons.Add(table.InCombo ? null : on); + groupBases.Add(null); + } + return; + } + + // Without an alias a derived table can only be the one being written, which this one cannot be. + string alias = sq.Alias ?? throw new NotSupportedException( + "Operation must use an updateable query: a derived table written to must select from one table, " + + "with no grouping or DISTINCT."); var (columns, rows) = ExecuteDerivedSource(sq.Query, alias); tables.Add(new SourceTable(alias, null, columns, rows)); kinds.Add(kind); ons.Add(on); + groupBases.Add(null); } void EmitLateral(SubqueryTable sq, JoinKind kind, Expression? on) @@ -1010,9 +1057,10 @@ void EmitLateral(SubqueryTable sq, JoinKind kind, Expression? on) plan, new EvalScope(outerColumns, new object?[outerColumns.Count], null)); tables.Add(new SourceTable( - alias, null, columns.Select(c => new OutputColumn(alias, c.Name, c.ClrType)).ToList(), null, plan)); + alias, null, columns.Select(c => c with { Qualifier = alias }).ToList(), null, plan)); kinds.Add(kind); ons.Add(on); + groupBases.Add(null); } void Walk(TableReference r, JoinKind kind, Expression? on) @@ -1044,6 +1092,10 @@ void WalkJoin(JoinTable j) Walk(j.Right, j.Kind, j.On); break; + case JoinShape.Conditional when j.Right is JoinTable group: + WalkGroup(j, group); + break; + // RIGHT JOIN keeps the right side: model it as the right side preserved (the base) with the left // side LEFT-joined onto it. The SET/DELETE target is usually that left side — which then becomes // the nullable side, so a right row with no match yields a null target that the WHERE drops. @@ -1070,8 +1122,166 @@ void WalkJoin(JoinTable j) } } + // A join onto a bracketed group — UPDATE a, b INNER JOIN c ON … parses as a CROSS (b INNER c) — lays the + // group's tables out after the left side's, which gives the same rows in the shapes ACE accepts (verified): + // - an inner or cross join whose ON reads only the tables before the group's first outer join, checked once + // those have joined (a INNER JOIN (b LEFT JOIN c) ON a.x = b.x); + // - a LEFT join onto a table followed only by LEFT joins, whose ON reads only that first table; when the + // first table is null-extended so is the rest of the group (a LEFT JOIN (b LEFT JOIN c) ON a.x = b.x). + // ACE refuses the others ("JOIN expression not supported"), a LEFT join onto an inner join among them. + void WalkGroup(JoinTable j, JoinTable group) + { + Walk(j.Left, JoinKind.Inner, null); + int first = tables.Count; + Walk(group, JoinKind.Inner, null); + int end = tables.Count; + + // The group's tables as units: a writable derived table's run is one, joined and null-extended whole + // (verified vs ACE, which accepts a derived join anywhere a table can be in these shapes). + List units = Enumerable.Range(first, end - first).Where(t => !tables[t].InCombo).ToList(); + int innerUnits = 1; + while (innerUnits < units.Count && kinds[units[innerUnits]] is JoinKind.Inner or JoinKind.Cross) + innerUnits++; + int innerEnd = innerUnits < units.Count ? units[innerUnits] : end; + bool left = j.Kind == JoinKind.Left; + HashSet readable = tables.Take(left ? first + tables[first].ComboWidth : innerEnd) + .Select(t => t.Alias).ToHashSet(StringComparer.OrdinalIgnoreCase); + bool onFits = j.On is null || Planning.IndexSelection.ReferencesOnly(j.On, readable); + + if (!left && onFits) + { + int last = units[innerUnits - 1]; + if (j.On is not null) + { + ons[last] = ons[last] is { } own ? new BinaryExpression(BinaryOperator.And, own, j.On) : j.On; + kinds[last] = JoinKind.Inner; + } + } + else if (left && onFits && innerUnits == 1 && units.Skip(1).All(u => kinds[u] == JoinKind.Left)) + { + kinds[first] = JoinKind.Left; + ons[first] = j.On; + for (int i = first + tables[first].ComboWidth; i < end; i++) + groupBases[i] ??= first; + } + else + { + throw new NotSupportedException($"UPDATE/DELETE over a {j.Kind} join onto this group is not supported."); + } + } + Walk(from, JoinKind.Inner, null); - return (tables, kinds, ons); + return (tables, kinds, ons, groupBases); + } + + /// + /// A derived table an UPDATE or DELETE can write through, as ACE allows (verified: UPDATE (SELECT TOP 2 * FROM t + /// ORDER BY Id DESC) SET x = 1 updates those two rows of t, and DELETE * FROM (SELECT …) deletes them): a SELECT + /// filtered, ordered and cut by its TOP or OFFSET, with no grouping or DISTINCT, over a table or a join. Its rows + /// are its tables' rows, so they can be written back: it becomes one source table per table it reads, whose + /// rows come together, as the query chose them. Without an alias it takes the name of the one table it reads. + /// Only the columns it selects can be read or set, under the names it gives them (verified vs ACE: SELECT X + /// AS Y makes SET Y write X, and a column not selected is unknown). A computed item neither stops the rest being + /// written nor can itself be read. Null for any other query. + /// + private List? WritableDerivedTable(SubqueryTable sq) + { + if (sq.Query is not SelectStatement + { + From: NamedTable or JoinTable, GroupBy.Count: 0, Having: null, Distinct: false, Into: null, + } select + || select.Projection.Any(item => item.Value is StarExpression or QualifiedStarExpression && item.Alias is not null)) + return null; + + var (inner, kinds, ons, groupBases) = ResolveSource(select.From); + var innerColumns = inner.SelectMany(t => t.Columns).ToList(); + + // The name each column goes by, per table and by position; a column the query does not select has none. + var names = inner.Select(t => new string?[t.Columns.Count]).ToArray(); + void Name(int table, int column, string name) => names[table][column] = name; + if (select.IsSelectStar) + for (int t = 0; t < inner.Count; t++) + for (int c = 0; c < inner[t].Columns.Count; c++) Name(t, c, inner[t].Columns[c].Name); + foreach (SelectItem item in select.Projection) + { + if (item.Value is ColumnReference reference) + { + // The one table's column it names; an ambiguous or unknown one exposes nothing. + var found = FindColumns(inner, reference.Table, reference.Column); + if (found.Count == 1) + Name(found[0].Table, found[0].Position, item.Alias ?? inner[found[0].Table].Columns[found[0].Position].Name); + continue; + } + for (int t = 0; t < inner.Count; t++) + { + if (item.Value is StarExpression + || item.Value is QualifiedStarExpression star && string.Equals(star.Table, inner[t].Alias, StringComparison.OrdinalIgnoreCase)) + { + for (int c = 0; c < inner[t].Columns.Count; c++) names[t][c] ??= inner[t].Columns[c].Name; + } + } + } + ExpressionEvaluator Over(object?[] values) => + new(new EvalScope(innerColumns, values, null), _scalarRunner, _parameters, _session); + + var rows = JoinRows(inner, kinds, ons, groupBases, select.Where, innerColumns); + object?[] Flat((RowId Id, object?[] Values)[] combo) => combo.SelectMany(c => c.Values).ToArray(); + if (select.OrderBy.Count > 0) + { + // Stable, as a SELECT's ORDER BY is: rows with equal keys keep the order they were read in. + rows = rows + .Select(row => (Row: row, Keys: select.OrderBy.Select(o => Over(Flat(row)).Evaluate(o.Value)).ToArray())) + .OrderBy(keyed => keyed.Keys, Comparer.Create((a, b) => + { + for (int i = 0; i < a.Length; i++) + { + int order = ExpressionEvaluator.CompareForSort(a[i], b[i]); + if (order != 0) + return select.OrderBy[i].Direction == SortDirection.Descending ? -order : order; + } + return 0; + })) + .Select(keyed => keyed.Row) + .ToList(); + } + + var counts = new ExpressionEvaluator(new EvalScope([], [], null), _scalarRunner, _parameters, _session); + int Count(Expression e) => Convert.ToInt32(counts.Evaluate(e), System.Globalization.CultureInfo.InvariantCulture); + if (select.Offset is { } offset) + rows = rows.Skip(Math.Max(0, Count(offset))).ToList(); + if (select.Top is { } top) + { + // TOP n PERCENT takes ceil(rows × n / 100), as a SELECT does. + int n = Count(top); + rows = rows.Take(select.TopPercent ? (int)(((long)rows.Count * n + 99) / 100) : Math.Max(0, n)).ToList(); + } + + // A column without a name stays in the row, where writing it back needs it, under a name no SQL can write. + string alias = sq.Alias ?? (inner.Count == 1 ? inner[0].Alias : "\0derived"); + return inner.Select((t, i) => t with + { + Alias = alias, + Key = $"{alias}\0{t.CacheKey}", + Columns = t.Columns.Select((c, position) => c with { Qualifier = alias, Name = names[i][position] ?? HiddenColumn }).ToList(), + Combos = i == 0 ? rows : null, + ComboWidth = i == 0 ? inner.Count : 1, + InCombo = i > 0, + }).ToList(); + } + + private const string HiddenColumn = "\0hidden"; + + /// The table and position of every source column named : among the tables + /// names — by alias, or failing that by table name — or among all of them. + private static List<(int Table, int Position)> FindColumns(List tables, string? qualifier, string column) + { + IEnumerable candidates = qualifier is null ? Enumerable.Range(0, tables.Count) : NamedTables(tables, qualifier); + var found = new List<(int, int)>(); + foreach (int t in candidates) + for (int c = 0; c < tables[t].Columns.Count; c++) + if (string.Equals(tables[t].Columns[c].Name, column, StringComparison.OrdinalIgnoreCase)) + found.Add((t, c)); + return found; } /// Runs a derived-table subquery (uncorrelated — a FROM/JOIN source) through the full query @@ -1095,7 +1305,8 @@ void WalkJoin(JoinTable j) /// value accumulates across matches — matching Access (e.g. a "one"-side counter incremented per match). /// private List<(RowId Id, object?[] Values)[]> JoinRows( - List tables, List kinds, List ons, Expression? where, IReadOnlyList columns) + List tables, List kinds, List ons, List groupBases, Expression? where, + IReadOnlyList columns) { // A physical row's value array is shared across every join row it appears in (see the method summary). var cache = new Dictionary<(string, RowId), object?[]>(); @@ -1129,6 +1340,19 @@ void WalkJoin(JoinTable j) var result = new List<(RowId, object?[])[]>(); var acc = new (RowId, object?[])[tables.Count]; + var nullExtended = new bool[tables.Count]; + + // A writable derived table's run of tables is null-extended, like its rows are joined, together. + void NullExtend(int i) + { + int width = tables[i].ComboWidth; + for (int k = i; k < i + width; k++) + { + acc[k] = (default, new object?[tables[k].Columns.Count]); + nullExtended[k] = true; + } + Recurse(i + width); + } bool Holds(Expression? predicate, int depth) => predicate is null || new ExpressionEvaluator( @@ -1144,6 +1368,38 @@ void Recurse(int i) return; } + // A table inside a LEFT-joined group has no row when the group's first table has none. + if (groupBases[i] is { } groupBase && nullExtended[groupBase]) + { + NullExtend(i); + return; + } + nullExtended[i] = false; + + // A writable derived table's rows, already joined, filtered, ordered and cut: each fills its run of + // tables at once, and this table's ON is checked once all of them are in. + if (tables[i].Combos is { } combos) + { + int width = tables[i].ComboWidth; + bool comboMatched = false; + foreach ((RowId, object?[])[] combo in combos) + { + for (int k = 0; k < width; k++) + { + acc[i + k] = combo[k]; + nullExtended[i + k] = false; + } + if (Holds(ons[i], i + width)) + { + comboMatched = true; + Recurse(i + width); + } + } + if (kinds[i] is JoinKind.Left or JoinKind.OuterApply && !comboMatched) + NullExtend(i); + return; + } + // A derived table's rows are already materialised and have no RowId (never a target). A physical // table is seeked when its ON allows (index-nested-loop), else scanned. IEnumerable<(RowId Id, object?[] Values)> rows; @@ -1177,7 +1433,7 @@ void Recurse(int i) { // Share a physical row's array across the combos it appears in (counter-accumulation semantics); // a derived row has no identity to share on, so use it directly. - acc[i] = tables[i].Table is null ? (id, values) : (id, Shared(tables[i].Alias, id, values)); + acc[i] = tables[i].Table is null ? (id, values) : (id, Shared(tables[i].CacheKey, id, values)); if (Holds(ons[i], i + 1)) { matched = true; @@ -1188,10 +1444,7 @@ void Recurse(int i) // LEFT join, and OUTER APPLY for the same reason: an outer row with no matching inner row is still // emitted, with the inner side all-null. if (kinds[i] is JoinKind.Left or JoinKind.OuterApply && !matched) - { - acc[i] = (default, new object?[tables[i].Columns.Count]); - Recurse(i + 1); - } + NullExtend(i); } Recurse(0); @@ -1211,7 +1464,8 @@ void Recurse(int i) /// expression (to seek it per outer row); else null (scan it). private static (IndexDef Index, Expression Key)? SeekPlanFor(int i, List tables, Expression? on) { - if (on is null || tables[i].Table is null) return null; // a derived table has no index to seek + // A derived table has no index to seek, and a writable one's rows are already chosen. + if (on is null || tables[i].Table is null || tables[i].Combos is not null || tables[i].InCombo) return null; TableDef def = tables[i].Table!.Definition; string alias = tables[i].Alias; HashSet earlier = tables.Take(i).Select(t => t.Alias).ToHashSet(StringComparer.OrdinalIgnoreCase); @@ -1241,31 +1495,34 @@ private static (IndexDef Index, Expression Key)? MatchSeek( IndexDef? index = def.Indexes.FirstOrDefault(ix => ix.RootPage > 0 && ix.Columns.Count == 1 && string.Equals(ix.Columns[0].Column.Name, c.Column, StringComparison.OrdinalIgnoreCase)); - return index is not null && ReferencesOnly(keySide, earlier) ? (index, keySide) : null; + return index is not null && IndexSelection.ReferencesOnly(keySide, earlier) ? (index, keySide) : null; } - private static bool ReferencesOnly(Expression e, HashSet aliases) => e switch + /// The tables a qualifier names: by alias, or failing that by table name. + private static List NamedTables(List tables, string qualifier) { - ColumnReference { Table: { } t } => aliases.Contains(t), - ColumnReference => false, // unqualified — can't attribute it to an earlier table safely - LiteralExpression or ParameterExpression or SystemVariableExpression => true, - BinaryExpression b => ReferencesOnly(b.Left, aliases) && ReferencesOnly(b.Right, aliases), - UnaryExpression u => ReferencesOnly(u.Operand, aliases), - FunctionCall f => f.Arguments.All(a => ReferencesOnly(a, aliases)), - _ => false, - }; + var byAlias = Enumerable.Range(0, tables.Count) + .Where(t => string.Equals(tables[t].Alias, qualifier, StringComparison.OrdinalIgnoreCase)).ToList(); + return byAlias.Count > 0 + ? byAlias + : Enumerable.Range(0, tables.Count) + .Where(t => string.Equals(tables[t].Table?.Name, qualifier, StringComparison.OrdinalIgnoreCase)).ToList(); + } - /// The source-table index a SET assignment (or a delete target) applies to: the alias/table-name - /// qualifier if given, else the single table (ambiguous when there are several). - private static int TargetIndex(List tables, string? qualifier, string what) + /// The source table a DELETE removes rows from: the one its target.* names, or the only one (a + /// join without a target is ambiguous). A writable derived table over a join is never one, as in ACE (verified: + /// "Could not delete from specified tables"). + private static int DeleteTarget(List tables, string? qualifier) { + List named = qualifier is null ? [.. Enumerable.Range(0, tables.Count)] : NamedTables(tables, qualifier); + if (named.Any(t => tables[t].ComboWidth > 1 || tables[t].InCombo)) + throw new InvalidOperationException("Could not delete from specified tables."); if (qualifier is null) return tables.Count == 1 ? 0 - : throw new InvalidOperationException($"{what} must be table-qualified when the statement joins several tables."); - int i = tables.FindIndex(t => - string.Equals(t.Alias, qualifier, StringComparison.OrdinalIgnoreCase) || - string.Equals(t.Table?.Name, qualifier, StringComparison.OrdinalIgnoreCase)); - return i >= 0 ? i : throw new InvalidOperationException($"{what} '{qualifier}' is not one of the statement's tables."); + : throw new InvalidOperationException("DELETE target must be table-qualified when the statement joins several tables."); + return named.Count > 0 + ? named[0] + : throw new InvalidOperationException($"DELETE target '{qualifier}' is not one of the statement's tables."); } /// The physical a SET/DELETE targets — a derived table (subquery source) has no @@ -1275,39 +1532,73 @@ private static Table TargetTable(List tables, int ti) => /// /// Executes UPDATE tableexpression SET col = expr, … [WHERE criteria]. The table expression may be a join, - /// and each SET target may name a specific joined table (Access's multi-table update). Each SET expression - /// may reference the current values; the WHERE is an ordinary expression (correlated EXISTS included). - /// Rows are rewritten in place (row id preserved). @@ROWCOUNT = matched join rows. + /// and each SET target may name a specific joined table (Access's multi-table update). The WHERE is an + /// ordinary expression (correlated EXISTS included). Rows are rewritten in place (row id preserved). + /// @@ROWCOUNT = matched join rows. + /// Every SET expression reads the joined row as it was before any of its SETs, so SET x = y, y = x + /// swaps the two (verified vs ACE). A target row that several joined rows match is updated once per match, each + /// match reading what the one before left (verified vs ACE: SET a.x = a.x + 1 over two matches adds 2). A column + /// set twice is an error, as in ACE ("Duplicate output destination"). + /// A SET on the side of an outer join that a joined row has no match in writes a new row into that table, + /// one per such joined row, holding the SET values and the table's defaults (verified vs ACE: SET b.v = 3 over + /// a LEFT JOIN b adds a row with v = 3 for each unmatched row of the left table, even when the value is + /// Null). /// private int ExecuteUpdate(UpdateStatement statement) { - var (tables, kinds, ons) = ResolveSource(statement.From); + var (tables, kinds, ons, groupBases) = ResolveSource(statement.From); var columns = tables.SelectMany(t => t.Columns).ToList(); - List<(RowId Id, object?[] Values)[]> joinRows = JoinRows(tables, kinds, ons, statement.Where, columns); + List<(RowId Id, object?[] Values)[]> joinRows = JoinRows(tables, kinds, ons, groupBases, statement.Where, columns); // Resolve each assignment to its (table index, column) once. var targets = statement.Assignments.Select(a => { - int ti = TargetIndex(tables, a.Table, "UPDATE SET column"); + // By the name the source gives the column, which a writable derived table may change or leave out. An + // unqualified name is looked for in every table, as ACE does, and must be in exactly one. + var found = FindColumns(tables, a.Table, a.Column); + if (found.Count > 1) + throw new InvalidOperationException( + $"The specified field '{a.Column}' could refer to more than one table listed in the FROM clause."); + if (found.Count == 0) + throw new InvalidOperationException(a.Table is not null && NamedTables(tables, a.Table).Count == 0 + ? $"UPDATE SET column '{a.Table}' is not one of the statement's tables." + : $"Column '{(a.Table is null ? "" : a.Table + ".")}{a.Column}' does not exist."); + (int ti, int position) = found[0]; Table tt = TargetTable(tables, ti); - ColumnDef col = tt.Definition.FindColumn(a.Column) - ?? throw new InvalidOperationException($"Column '{a.Column}' does not exist in '{tt.Name}'."); + ColumnDef col = tt.Definition.Columns[position]; return (TableIndex: ti, Column: col, a.Value); }).ToList(); + if (targets.GroupBy(t => (t.TableIndex, t.Column.Index)).FirstOrDefault(g => g.Count() > 1) is { } duplicate) + throw new InvalidOperationException( + $"Duplicate output destination '{tables[duplicate.Key.TableIndex].Alias}.{duplicate.First().Column.Name}'."); // Apply SETs to the shared value arrays; snapshot each touched row's original bytes on first touch. var dirty = new Dictionary<(string, RowId), (Table Table, RowId Id, object?[] Original, object?[] Values)>(); + // A null-extended side's value array is the joined row's own, so it keys the new row it becomes. + var newRows = new Dictionary Provided)>(ReferenceEqualityComparer.Instance); foreach (var combo in joinRows) { - foreach ((int ti, ColumnDef col, Expression valueExpr) in targets) + // Every SET of this joined row reads the row as it was before any of them. + object?[] flat = combo.SelectMany(c => c.Values).ToArray(); + var eval = new ExpressionEvaluator(new EvalScope(columns, flat, null), _scalarRunner, _parameters, _session); + object?[] results = targets.Select(t => eval.Evaluate(t.Value)).ToArray(); + + for (int i = 0; i < targets.Count; i++) { + (int ti, ColumnDef col, _) = targets[i]; object?[] shared = combo[ti].Values; - var key = (tables[ti].Alias, combo[ti].Id); - if (!dirty.ContainsKey(key)) dirty[key] = (TargetTable(tables, ti), combo[ti].Id, (object?[])shared.Clone(), shared); - - object?[] flat = combo.SelectMany(c => c.Item2).ToArray(); - var eval = new ExpressionEvaluator(new EvalScope(columns, flat, null), _scalarRunner, _parameters, _session); - shared[col.Index] = eval.Evaluate(valueExpr); + if (IsNullExtended(combo[ti])) + { + if (!newRows.TryGetValue(shared, out var newRow)) + newRows[shared] = newRow = (TargetTable(tables, ti), []); + newRow.Provided.Add(col.Index); + } + else + { + var key = (tables[ti].CacheKey, combo[ti].Id); + if (!dirty.ContainsKey(key)) dirty[key] = (TargetTable(tables, ti), combo[ti].Id, (object?[])shared.Clone(), shared); + } + shared[col.Index] = results[i]; } } @@ -1353,34 +1644,48 @@ private int ExecuteUpdate(UpdateStatement statement) table.MoveIndexEntry(index, original, values, id); } + foreach (var (values, (table, provided)) in newRows) + InsertNewRow(table.Name, table, DefaultsOf(table.Definition), values, provided); + int affected = joinRows.Count; if (_session is not null) _session.RowCount = affected; return affected; } + /// Whether a joined row's entry for a table is the null row an outer join supplies where the table has + /// no match. Row id 0/0 is never a data row: page 0 is the database header. + private static bool IsNullExtended((RowId Id, object?[] Values) entry) => entry.Id.Equals(default(RowId)); + /// /// Executes DELETE [target.*] FROM tableexpression [WHERE criteria]. The table expression may be a join; /// target.* selects which joined table's rows to delete (defaults to the single table). Each /// matched target row's index entries are removed and the row is soft-deleted (row id kept, TDEF row - /// count decremented). @@ROWCOUNT = distinct rows deleted. + /// count decremented). @@ROWCOUNT = distinct rows deleted, plus the joined rows where an outer join left the + /// target without a row, which delete nothing but which ACE counts (verified). /// private int ExecuteDelete(DeleteStatement statement) { - var (tables, kinds, ons) = ResolveSource(statement.From); + var (tables, kinds, ons, groupBases) = ResolveSource(statement.From); var columns = tables.SelectMany(t => t.Columns).ToList(); - List<(RowId Id, object?[] Values)[]> joinRows = JoinRows(tables, kinds, ons, statement.Where, columns); + List<(RowId Id, object?[] Values)[]> joinRows = JoinRows(tables, kinds, ons, groupBases, statement.Where, columns); - int ti = TargetIndex(tables, statement.TargetTable, "DELETE target"); + int ti = DeleteTarget(tables, statement.TargetTable); Table target = TargetTable(tables, ti); var deleted = new Dictionary(); + int withoutRow = 0; foreach (var combo in joinRows) - deleted.TryAdd(combo[ti].Id, combo[ti].Values); // one delete per distinct target row + { + if (IsNullExtended(combo[ti])) + withoutRow++; + else + deleted.TryAdd(combo[ti].Id, combo[ti].Values); // one delete per distinct target row + } // Delete the distinct target rows and everything ON DELETE CASCADE reaches, children before parents. CascadeDelete(target, deleted.Select(kv => (kv.Key, kv.Value))); - int affected = deleted.Count; + int affected = deleted.Count + withoutRow; if (_session is not null) _session.RowCount = affected; return affected; } diff --git a/src/LibRed/LibRed.Engine/Execution/SubqueryHoisting.cs b/src/LibRed/LibRed.Engine/Execution/SubqueryHoisting.cs index 26842889c..c6d6b9ccc 100644 --- a/src/LibRed/LibRed.Engine/Execution/SubqueryHoisting.cs +++ b/src/LibRed/LibRed.Engine/Execution/SubqueryHoisting.cs @@ -67,14 +67,10 @@ internal static bool MayReferenceOuter(Expression expression, HashSet ou ColumnReference => false, LiteralExpression or ParameterExpression or SystemVariableExpression or StarExpression or QualifiedStarExpression => false, - BinaryExpression b => HasUnqualifiedColumn(b.Left) || HasUnqualifiedColumn(b.Right), - UnaryExpression u => HasUnqualifiedColumn(u.Operand), - FunctionCall f => f.Arguments.Any(HasUnqualifiedColumn), - InListExpression il => HasUnqualifiedColumn(il.Value) || il.Items.Any(HasUnqualifiedColumn), ScalarSubquery sq => StatementHasUnqualified(sq.Query), ExistsExpression ex => StatementHasUnqualified(ex.Query), InSubqueryExpression isq => HasUnqualifiedColumn(isq.Value) || StatementHasUnqualified(isq.Query), - _ => true, // unknown shape: assume the worst + _ => expression.Operands()?.Any(HasUnqualifiedColumn) ?? true, // unknown shape: assume the worst }; private static bool StatementHasUnqualified(SqlStatement statement) @@ -133,13 +129,9 @@ private static bool Statement(SqlStatement statement, HashSet outer) ColumnReference => false, LiteralExpression or ParameterExpression or SystemVariableExpression or StarExpression or QualifiedStarExpression => false, - BinaryExpression b => Expr(b.Left, outer) || Expr(b.Right, outer), - UnaryExpression u => Expr(u.Operand, outer), - FunctionCall f => f.Arguments.Any(a => Expr(a, outer)), - InListExpression il => Expr(il.Value, outer) || il.Items.Any(i => Expr(i, outer)), ScalarSubquery sq => Statement(sq.Query, outer), ExistsExpression ex => Statement(ex.Query, outer), InSubqueryExpression isq => Expr(isq.Value, outer) || Statement(isq.Query, outer), - _ => true, // unknown expression shape + _ => e.Operands()?.Any(o => Expr(o, outer)) ?? true, // unknown expression shape }; } diff --git a/src/LibRed/LibRed.Engine/Execution/VbaDateText.cs b/src/LibRed/LibRed.Engine/Execution/VbaDateText.cs new file mode 100644 index 000000000..f2275d5c6 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/VbaDateText.cs @@ -0,0 +1,322 @@ +using System.Globalization; + +namespace LibRed.Engine.Execution; + +/// +/// Text read as a date and time the way CDate reads it — OLE Automation's own parser rather than .NET's +/// (verified vs ACE; documented at +/// https://learn.microsoft.com/en-us/office/vba/Language/Concepts/Getting-Started/type-conversion-functions). +/// +/// +/// +/// The text is numbers and words split by separators. / - ,, spaces and the regional date +/// separator split a date; : . and the regional time separator split a time, so 2.5 is 02:05. +/// A regional date separator of . makes it a date separator instead. +/// A time is two or three numbers split by time separators, or one number with AM or PM; hours run to 23 and +/// minutes and seconds to 59. AM or PM moves an hour up to 12 (12 AM is midnight) and leaves a later one alone. +/// A time may come before or after the date. +/// A date of two numbers takes the regional order of day and month, then the other order, then month and year +/// (1,2020 is January 2020); with no year it is in the current year. Three numbers are year, month and day +/// when the first cannot be a day, otherwise in the regional order, then with day and month swapped. +/// A month name, full or abbreviated, takes one number as its day, or as its year when it cannot be one +/// (Feb 30 is 2030-02-01), or two as day and year. A day-of-week name is not recognised. +/// A year under 100 is placed by the calendar's two-digit window (0 is 2000, 99 1999); a year past +/// 9999 is invalid. +/// Anything else — one bare number, a stray separator, an unknown word — is not a date, and the caller reads the +/// text as a number instead. +/// +/// +internal static class VbaDateText +{ + private enum Kind { Number, Month, Meridiem, DateSeparator, TimeSeparator } + + /// A part of the text. A number keeps how many digits it was written with; a time separator is + /// when it was a period. + private readonly record struct Token(Kind Kind, int Value, int Digits = 0); + + private const int Dot = 1; + + public static bool TryParse(string text, CultureInfo culture, out DateTime value) + { + value = default; + if (Tokenize(text, culture) is not { } tokens) + return false; + + TimeSpan? time = null; + var dateNumbers = new List(); + int? month = null; + Token? separator = null; // the separator just read, if the last token was one + + for (int i = 0; i < tokens.Count; i++) + { + Token token = tokens[i]; + switch (token.Kind) + { + case Kind.Number when IsTimeStart(tokens, i): + // A time follows spacing or starts the text; an explicit date separator cannot lead into one. + if (time is not null || separator is { Value: not Spacing } || ReadTime(tokens, ref i) is not { } read) + return false; + time = read; + separator = null; + break; + case Kind.Number: + dateNumbers.Add(token.Value); + separator = null; + break; + case Kind.Month: + if (month is not null) + return false; + month = token.Value; + separator = null; + break; + case Kind.DateSeparator: + // Between two parts only: not first, not doubled, not last. + if (separator is not null || i == 0 || i == tokens.Count - 1) + return false; + separator = token; + break; + default: + return false; + } + } + + DateTime date; + if (month is null && dateNumbers.Count == 0) + { + if (time is null) + return false; + date = new DateTime(1899, 12, 30); + } + else if (DateOf(dateNumbers, month, culture) is { } parsed) + { + date = parsed; + } + else + { + return false; + } + + value = date + (time ?? TimeSpan.Zero); + return true; + } + + /// The words, numbers and separators of the text, or null when it holds anything else. + private static List? Tokenize(string text, CultureInfo culture) + { + DateTimeFormatInfo format = culture.DateTimeFormat; + bool dotIsDate = format.DateSeparator == "."; + var tokens = new List(); + int i = 0; + while (i < text.Length) + { + char c = text[i]; + if (char.IsWhiteSpace(c)) + { + // Spacing separates only two parts with nothing else between them. + int end = i; + while (end < text.Length && char.IsWhiteSpace(text[end])) end++; + if (tokens.Count > 0 && tokens[^1].Kind is Kind.Number or Kind.Month or Kind.Meridiem + && end < text.Length && char.IsLetterOrDigit(text[end])) + tokens.Add(new Token(Kind.DateSeparator, Spacing)); + i = end; + } + else if (char.IsDigit(c)) + { + int start = i; + while (i < text.Length && char.IsDigit(text[i])) i++; + if (i - start > 9) + return null; + tokens.Add(new Token(Kind.Number, int.Parse(text.AsSpan(start, i - start), CultureInfo.InvariantCulture), i - start)); + } + else if (char.IsLetter(c)) + { + int start = i; + while (i < text.Length && char.IsLetter(text[i])) i++; + string word = text[start..i]; + if (Meridiem(word, format) is { } pm) + tokens.Add(new Token(Kind.Meridiem, pm ? 1 : 0)); + else if (MonthOf(word, format) is { } m) + tokens.Add(new Token(Kind.Month, m)); + else + return null; + } + else if (c == '.' && (!dotIsDate || AfterSeconds(tokens)) + || c == ':' || format.TimeSeparator.Length == 1 && c == format.TimeSeparator[0]) + { + DropSpacing(tokens); + tokens.Add(new Token(Kind.TimeSeparator, c == '.' ? Dot : 0)); + i++; + SkipSpacing(text, ref i); + } + else if (c is '/' or '-' or ',' || format.DateSeparator.Length == 1 && c == format.DateSeparator[0]) + { + DropSpacing(tokens); + tokens.Add(new Token(Kind.DateSeparator, 0)); + i++; + SkipSpacing(text, ref i); + } + else + { + return null; + } + } + return tokens; + } + + /// The value of a date separator that is only spacing, which a time may follow. + private const int Spacing = 1; + + /// Whether the tokens so far end in hours, minutes and seconds written with colons, so a period can only + /// start their fraction. + private static bool AfterSeconds(List tokens) => + tokens.Count >= 5 + && tokens[^1].Kind == Kind.Number && tokens[^2] is { Kind: Kind.TimeSeparator, Value: not Dot } + && tokens[^3].Kind == Kind.Number && tokens[^4] is { Kind: Kind.TimeSeparator, Value: not Dot } + && tokens[^5].Kind == Kind.Number; + + /// An explicit separator absorbs the spacing before it. + private static void DropSpacing(List tokens) + { + if (tokens.Count > 0 && tokens[^1] is { Kind: Kind.DateSeparator, Value: Spacing }) + tokens.RemoveAt(tokens.Count - 1); + } + + private static void SkipSpacing(string text, ref int i) + { + while (i < text.Length && char.IsWhiteSpace(text[i])) i++; + } + + /// Whether the number at starts a time: a time separator or AM/PM follows it. + private static bool IsTimeStart(List tokens, int i) + { + int next = i + 1; + if (next < tokens.Count && tokens[next].Kind == Kind.DateSeparator && next + 1 < tokens.Count + && tokens[next + 1].Kind == Kind.Meridiem) + next++; // "3 PM": the spacing before the designator + return next < tokens.Count && tokens[next].Kind is Kind.TimeSeparator or Kind.Meridiem; + } + + /// The time starting at , leaving on its last token; null when + /// it is not a valid time. + private static TimeSpan? ReadTime(List tokens, ref int i) + { + var parts = new List { tokens[i].Value }; + long fraction = 0; + bool dotted = false; + while (i + 2 < tokens.Count && tokens[i + 1].Kind == Kind.TimeSeparator && tokens[i + 2].Kind == Kind.Number) + { + // A LibRed extension: after hours, minutes and seconds written with colons, a period and up to seven digits + // are a fraction of a second, as LibRed's own SQL writes a time with milliseconds + // (TIMEVALUE('12:30:45.123')). ACE refuses a fourth part. + if (parts.Count == 3 && !dotted && tokens[i + 1].Value == Dot && tokens[i + 2].Digits <= 7) + { + fraction = tokens[i + 2].Value * (long)Math.Pow(10, 7 - tokens[i + 2].Digits); + i += 2; + break; + } + dotted |= tokens[i + 1].Value == Dot; + parts.Add(tokens[i + 2].Value); + i += 2; + } + if (i + 1 < tokens.Count && tokens[i + 1].Kind == Kind.TimeSeparator) + return null; // a separator with no number after it + + bool? pm = null; + int next = i + 1; + if (next < tokens.Count && tokens[next].Kind == Kind.DateSeparator && next + 1 < tokens.Count + && tokens[next + 1].Kind == Kind.Meridiem) + next++; + if (next < tokens.Count && tokens[next].Kind == Kind.Meridiem) + { + pm = tokens[next].Value == 1; + i = next; + } + + if (parts.Count > 3 || parts.Count == 1 && pm is null) + return null; + int hour = parts[0], minute = parts.Count > 1 ? parts[1] : 0, second = parts.Count > 2 ? parts[2] : 0; + if (hour > 23 || minute > 59 || second > 59) + return null; + if (pm is { } afternoon && hour <= 12) + hour = afternoon ? (hour == 12 ? 12 : hour + 12) : (hour == 12 ? 0 : hour); + return new TimeSpan(hour, minute, second) + TimeSpan.FromTicks(fraction); + } + + private static DateTime? DateOf(List numbers, int? month, CultureInfo culture) + { + int currentYear = DateTime.Today.Year; + if (month is { } m) + { + return numbers.Count switch + { + 1 => Make(currentYear, m, numbers[0]) ?? Make(Year(numbers[0], culture), m, 1), + 2 => Make(Year(numbers[1], culture), m, numbers[0]), + _ => null, + }; + } + + bool dayFirst = DayBeforeMonth(culture.DateTimeFormat.ShortDatePattern); + switch (numbers.Count) + { + case 2: + { + (int day, int mon) = dayFirst ? (numbers[0], numbers[1]) : (numbers[1], numbers[0]); + return Make(currentYear, mon, day) + ?? Make(currentYear, day, mon) + ?? Make(Year(numbers[1], culture), numbers[0], 1) + ?? Make(Year(numbers[0], culture), numbers[1], 1); + } + case 3: + { + if (numbers[0] > 31) + return Make(Year(numbers[0], culture), numbers[1], numbers[2]); + (int day, int mon, int year) = YearFirst(culture.DateTimeFormat.ShortDatePattern) + ? (numbers[2], numbers[1], numbers[0]) + : dayFirst ? (numbers[0], numbers[1], numbers[2]) : (numbers[1], numbers[0], numbers[2]); + int y = Year(year, culture); + return Make(y, mon, day) ?? Make(y, day, mon); + } + default: + return null; + } + } + + private static int Year(int year, CultureInfo culture) => + year < 100 ? culture.Calendar.ToFourDigitYear(year) : year; + + private static DateTime? Make(int year, int month, int day) => + year is >= 1 and <= 9999 && month is >= 1 and <= 12 && day >= 1 && day <= DateTime.DaysInMonth(year, month) + ? new DateTime(year, month, day) + : null; + + private static bool DayBeforeMonth(string pattern) + { + int d = pattern.IndexOf('d'), m = pattern.IndexOf('M'); + return d >= 0 && m >= 0 && d < m; + } + + private static bool YearFirst(string pattern) + { + int y = pattern.IndexOf('y'), m = pattern.IndexOf('M'); + return y >= 0 && m >= 0 && y < m; + } + + private static bool? Meridiem(string word, DateTimeFormatInfo format) => + Is(word, format.AMDesignator) || Is(word, "AM") ? false + : Is(word, format.PMDesignator) || Is(word, "PM") ? true + : null; + + private static int? MonthOf(string word, DateTimeFormatInfo format) + { + for (int i = 0; i < 12; i++) + { + if (Is(word, format.MonthNames[i]) || Is(word, format.AbbreviatedMonthNames[i]) + || Is(word, format.MonthGenitiveNames[i]) || Is(word, format.AbbreviatedMonthGenitiveNames[i])) + return i + 1; + } + return null; + } + + private static bool Is(string word, string name) => + name.Length > 0 && string.Equals(word, name.TrimEnd('.'), StringComparison.OrdinalIgnoreCase); +} diff --git a/src/LibRed/LibRed.Engine/Execution/WindowFunctions.cs b/src/LibRed/LibRed.Engine/Execution/WindowFunctions.cs index 13cd93cac..c44bdcafa 100644 --- a/src/LibRed/LibRed.Engine/Execution/WindowFunctions.cs +++ b/src/LibRed/LibRed.Engine/Execution/WindowFunctions.cs @@ -1,38 +1,47 @@ -namespace LibRed.Engine.Execution; +namespace LibRed.Engine.Execution; -/// -/// One partition of a window's input, its rows already in window order, together with the peer-group -/// information the ranking functions need. A peer group is a run of adjacent rows whose ORDER BY keys -/// compare equal; with no ORDER BY the whole partition is one peer group, which is what makes RANK constant -/// over an unordered window. -/// -/// Positions are indexes into the partition's own window order, not into the input. -internal sealed class WindowPartition( - IReadOnlyList peerStart, IReadOnlyList peerOrdinal, IReadOnlyList arguments) +/// Fills — one slot per row of the partition, in window order. +internal delegate void WindowEvaluator(WindowPartition partition, object?[] output); + +/// What a window function's declared type can depend on: its arguments' declared types. +internal interface IWindowTyping { - /// Rows in this partition. - public int Count => peerStart.Count; + /// The declared type of an argument; null when it is absent or its type is unknown. + Type? ArgumentType(int argument); - /// Position of the first row of the peer group holding . RANK is this - /// plus one, which is why ranks skip after a tie. - public int PeerStart(int position) => peerStart[position]; + /// The one type the given arguments' values share, as CASE's alternatives do; an absent argument or a + /// bare Null takes no part. Null when any other has an unknown type or the types cannot be reconciled. + Type? SharedType(params int[] arguments); +} - /// Zero-based ordinal of the peer group holding , counted from the - /// start of the partition. DENSE_RANK is this plus one, which is why it does not skip. - public int PeerOrdinal(int position) => peerOrdinal[position]; +/// What a window function may be written with, beyond its arguments and OVER (PARTITION BY … ORDER BY …). +[Flags] +internal enum WindowOptions +{ + None = 0, - /// The value of the call's th argument on the row at - /// — for the functions that take one (NTILE, LAG, FIRST_VALUE, …). - public object? Argument(int position, int argument) => arguments[position][argument]; -} + /// A frame clause: the function reads the frame. The standard gives the ranking and offset functions + /// none — they see the whole partition. + Frame = 1, -/// Fills — one slot per row of the partition, in window order. -internal delegate void WindowEvaluator(WindowPartition partition, object?[] output); + /// RESPECT NULLS or IGNORE NULLS. + NullTreatment = 2, + + /// FROM FIRST or FROM LAST. + FromLast = 4, -/// The declared CLR type of the result, given the declared type of the first argument -/// (null when there is none) — the window counterpart of QueryExecutor.DeclaredFunctionType. + /// DISTINCT, as the aggregates take it. + Distinct = 8, + + /// FILTER (WHERE …), as every aggregate takes it. + Filter = 16, +} + +/// The declared CLR type of the result — the window counterpart of +/// QueryExecutor.DeclaredFunctionType. The values a function returns are converted to it. internal sealed record WindowFunctionDef( - int MinArguments, int MaxArguments, Func ResultType, WindowEvaluator Evaluate); + int MinArguments, int MaxArguments, Func ResultType, WindowEvaluator Evaluate, + WindowOptions Options = WindowOptions.None); /// /// The window functions the engine implements. This table IS the extension point: because the grammar hangs @@ -41,6 +50,15 @@ internal sealed record WindowFunctionDef( /// internal static class WindowFunctions { + // The value on the frame's first or last row, Null for an empty frame. Over the default frame the first is the + // partition's, and the last is the current row's last peer — not the partition's last. Declared before Registry, + // which a static initializer reads in textual order. + private static readonly WindowFunctionDef FirstValue = new(1, 1, static t => t.ArgumentType(0), + static (p, o) => FrameValue(p, o, fromLast: false), WindowOptions.Frame | WindowOptions.NullTreatment); + + private static readonly WindowFunctionDef LastValue = new(1, 1, static t => t.ArgumentType(0), + static (p, o) => FrameValue(p, o, fromLast: true), WindowOptions.Frame | WindowOptions.NullTreatment); + private static readonly Dictionary Registry = new(StringComparer.OrdinalIgnoreCase) { // Position within the partition, 1-based. The only window function EF Core emits. @@ -54,15 +72,285 @@ internal static class WindowFunctions // The same, counting peer GROUPS rather than rows, so nothing is skipped. ["DENSE_RANK"] = new(0, 0, static _ => typeof(int), static (p, o) => { for (int i = 0; i < o.Length; i++) o[i] = p.PeerOrdinal(i) + 1; }), + + // (RANK − 1) / (rows − 1): the share of the partition ranked before the row, 0 for a partition of one. + ["PERCENT_RANK"] = new(0, 0, static _ => typeof(double), + static (p, o) => + { + for (int i = 0; i < o.Length; i++) + o[i] = o.Length == 1 ? 0.0 : (double)p.PeerStart(i) / (o.Length - 1); + }), + + // The share of the partition up to and including the row's last peer. + ["CUME_DIST"] = new(0, 0, static _ => typeof(double), + static (p, o) => { for (int i = 0; i < o.Length; i++) o[i] = (double)p.PeerEnd(i) / o.Length; }), + + ["NTILE"] = new(1, 1, static _ => typeof(int), Ntile), + ["LAG"] = new(1, 3, static t => t.SharedType(0, 2), static (p, o) => Offset(p, o, forward: false), + WindowOptions.NullTreatment), + ["LEAD"] = new(1, 3, static t => t.SharedType(0, 2), static (p, o) => Offset(p, o, forward: true), + WindowOptions.NullTreatment), + + ["FIRST_VALUE"] = FirstValue, + ["LAST_VALUE"] = LastValue, + ["NTH_VALUE"] = new(2, 2, static t => t.ArgumentType(0), NthValue, + WindowOptions.Frame | WindowOptions.NullTreatment | WindowOptions.FromLast), + + // The ordered-set aggregates over each frame; the parser has put the WITHIN GROUP key after the fraction. + ["PERCENTILE_CONT"] = PercentileOf("PERCENTILE_CONT"), + ["PERCENTILE_DISC"] = PercentileOf("PERCENTILE_DISC"), + ["LISTAGG"] = new(2, int.MaxValue, static _ => typeof(string), ListAggOf, + WindowOptions.Frame | WindowOptions.Distinct | WindowOptions.Filter), + + // Access's own First and Last, over the frame rather than the group: the same rows as FIRST_VALUE and + // LAST_VALUE, as the grouped forms take the group's first and last row. + ["FIRST"] = FirstValue, + ["LAST"] = LastValue, + + // Every aggregate RunningAggregate computes is a window function too — see Lookup. }; + /// + /// An aggregate over each row's frame — by default, with an ORDER BY, the partition's rows up to the current row + /// and its peers (a running total, ties sharing its value); without one, every row of the partition. The values + /// are the grouped aggregate's (), and so is the declared type; an empty frame gives + /// what an empty group does. With DISTINCT each frame's value counts once, however many of its rows carry it; a + /// binary set function (CORR, REGR_SLOPE, …) reads a pair from each row and takes no DISTINCT. + /// + private static WindowFunctionDef Aggregate(string name) => new( + RunningAggregate.IsPair(name) ? 2 : 1, + RunningAggregate.IsPair(name) ? 2 : 1, + typing => QueryExecutor.AggregateResultType(name, typing.ArgumentType(0)), + (p, o) => + { + WindowCall call = p.Call; + if (call.Star && name != "COUNT") + throw new InvalidOperationException($"{name}(*) is not an aggregate; only COUNT takes *."); + var frames = new FrameRows[o.Length]; + for (int i = 0; i < frames.Length; i++) + frames[i] = p.Frame(i); + if (frames.Length == 0) + return; + + // A frame that only ever grows at one end — every running total, the default frame included — is added + // to row by row rather than summed afresh: from the front when the frames share their first row, from + // the back when they share their last. + bool contiguous = frames.All(f => !f.HasExclusion); + if (contiguous && frames.All(f => f.Start == frames[0].Start) && Growing(frames, f => f.End)) + { + var aggregate = new FrameAggregate(name, p); + for (int i = 0, added = frames[0].Start; i < frames.Length; i++) + { + for (; added < frames[i].End; added++) + aggregate.Add(added); + o[i] = aggregate.Result; + } + } + else if (contiguous && frames.All(f => f.End == frames[0].End) && Growing(frames, f => f.Start)) + { + var aggregate = new FrameAggregate(name, p); + for (int i = frames.Length - 1, added = frames[0].End; i >= 0; i--) + { + for (; added > frames[i].Start; added--) + aggregate.Add(added - 1); + o[i] = aggregate.Result; + } + } + else + { + for (int i = 0; i < frames.Length; i++) + { + if (i > 0 && frames[i] == frames[i - 1]) + { + o[i] = o[i - 1]; + continue; + } + var aggregate = new FrameAggregate(name, p); + foreach (int position in frames[i].Positions()) + aggregate.Add(position); + o[i] = aggregate.Result; + } + } + }, + WindowOptions.Frame | WindowOptions.Filter | (RunningAggregate.IsPair(name) ? 0 : WindowOptions.Distinct)); + + /// A fed from a partition's rows; under DISTINCT each value goes in once, + /// equal as GROUP BY takes values to be equal. + private sealed class FrameAggregate(string name, WindowPartition p) + { + private readonly RunningAggregate _aggregate = new(name, countRows: p.Call.Star, currency: p.Call.Currency); + private readonly HashSet? _seen = p.Call.Distinct ? [] : null; + + public object? Result => _aggregate.Result; + + public void Add(int position) + { + if (!p.Includes(position)) + return; + if (p.ArgumentCount == 2) + { + _aggregate.AddPair(p.Argument(position, 0), p.Argument(position, 1)); + return; + } + object? value = p.Call.Star ? null : p.Argument(position, 0); + if (_seen is not null && (value is null || !_seen.Add(new QueryExecutor.GroupKey([value])))) + return; + _aggregate.Add(value); + } + } + + /// + /// PERCENTILE_CONT(fraction) WITHIN GROUP (ORDER BY key) and PERCENTILE_DISC over each row's frame — + /// by default the partition up to the row's last peer, or all of it; see . Each row's own + /// fraction applies. + /// + private static WindowFunctionDef PercentileOf(string name) => new(2, 2, + typing => Percentile.ResultType(name, typing.ArgumentType(1)), + (p, o) => + { + FrameRows previous = default; + for (int i = 0; i < o.Length; i++) + { + // Peers under the default frame, and every row of an unordered window, share their frame. + FrameRows frame = p.Frame(i); + o[i] = i > 0 && frame == previous && Equals(p.Argument(i, 0), p.Argument(i - 1, 0)) + ? o[i - 1] + : Percentile.Of(name, frame.Positions().Where(p.Includes).Select(k => p.Argument(k, 1)), + p.Argument(i, 0), p.Call.WithinGroup![0]); + previous = frame; + } + }, + WindowOptions.Frame | WindowOptions.Filter); + + /// + /// LISTAGG(x [, separator]) WITHIN GROUP (ORDER BY …) over each row's frame; see . The + /// arguments are the value, the separator when written, and the WITHIN GROUP keys. + /// + private static void ListAggOf(WindowPartition p, object?[] o) + { + IReadOnlyList directions = p.Call.WithinGroup!; + int keys = directions.Count; + string separator = p.ArgumentCount - keys == 2 && o.Length > 0 ? (string)p.Argument(0, 1)! : ""; + FrameRows previous = default; + for (int i = 0; i < o.Length; i++) + { + FrameRows frame = p.Frame(i); + o[i] = i > 0 && frame == previous + ? o[i - 1] + : ListAgg.Of( + frame.Positions().Where(p.Includes).Select(k => + (p.Argument(k, 0), Enumerable.Range(p.ArgumentCount - keys, keys).Select(a => p.Argument(k, a)).ToArray())), + separator, directions, p.Call.Distinct); + previous = frame; + } + } + + /// Whether a frame edge never moves back along the window order. + private static bool Growing(FrameRows[] frames, Func edge) + { + for (int i = 1; i < frames.Length; i++) + if (edge(frames[i]) < edge(frames[i - 1])) + return false; + return true; + } + + /// + /// NTILE(n): the partition cut in window order into n numbered buckets as even as can be, the + /// first ones taking the rows left over (10 rows in 4 buckets are 3, 3, 2, 2). Ties do not matter. The standard + /// makes n a constant, so the partition's first row gives it; read as CLng reads it. A Null n + /// gives Null, and one below 1 is an invalid procedure call. + /// + private static void Ntile(WindowPartition p, object?[] o) + { + if (o.Length == 0 || WholeArgument(p.Argument(0, 0), "NTILE's bucket count", least: 1) is not { } buckets) + return; + long size = o.Length / buckets, larger = o.Length % buckets; + long bucket = 1, filled = 0; + for (int i = 0; i < o.Length; i++) + { + if (filled == size + (bucket <= larger ? 1 : 0)) + { + bucket++; + filled = 0; + } + o[i] = (int)bucket; + filled++; + } + } + + /// + /// LAG(x [, offset [, default]]) and LEAD: x on the row offset rows before or after in + /// window order (1 when omitted; 0 is the row itself), or default (Null when omitted) where the partition + /// has no such row. Rows are counted one by one, ties or not — under IGNORE NULLS, only those where x is not + /// Null. Each row's own offset and default apply; a Null offset gives Null, and a negative one is an invalid + /// procedure call. + /// + private static void Offset(WindowPartition p, object?[] o, bool forward) + { + for (int i = 0; i < o.Length; i++) + { + long offset = 1; + if (p.ArgumentCount > 1) + { + if (WholeArgument(p.Argument(i, 1), "The offset", least: 0) is not { } given) + { + o[i] = null; + continue; + } + offset = given; + } + long target = p.Step(i, offset, forward); + o[i] = target >= 0 + ? p.Argument((int)target, 0) + : p.ArgumentCount > 2 ? p.Argument(i, 2) : null; + } + } + + /// + /// The first argument's value on the frame's first row or, , its last; Null for an + /// empty frame. By default Nulls are respected — a Null on that row is the result; under IGNORE NULLS it is the + /// first or last row whose value is not Null. + /// + private static void FrameValue(WindowPartition p, object?[] o, bool fromLast) + { + for (int i = 0; i < o.Length; i++) + o[i] = p.Pick(p.Frame(i), 0, fromLast) is var position and >= 0 ? p.Argument(position, 0) : null; + } + + /// + /// NTH_VALUE(x, n) [FROM FIRST | FROM LAST]: x on the frame's nth row, counted from its first + /// row or its last, or Null while the frame has fewer rows — so over the default frame the rows before the + /// nth peer group are Null. IGNORE NULLS counts only the rows where x is not Null. Each row's own + /// n applies, as LAG's offset does; a Null n gives Null, and one below 1 is an invalid procedure + /// call. + /// + private static void NthValue(WindowPartition p, object?[] o) + { + for (int i = 0; i < o.Length; i++) + o[i] = WholeArgument(p.Argument(i, 1), "NTH_VALUE's row number", least: 1) is { } n + && p.Pick(p.Frame(i), n - 1, p.Call.FromLast) is var position and >= 0 + ? p.Argument(position, 0) + : null; + } + + /// A whole-number argument, read as CLng reads it; null for Null, and an invalid procedure call below + /// . + private static long? WholeArgument(object? value, string what, long least) + { + if (value is null) + return null; + long n = Convert.ToInt64(ExpressionEvaluator.ConversionNumber(value), System.Globalization.CultureInfo.InvariantCulture); + return n >= least ? n : throw new ArgumentException($"Invalid procedure call: {what} cannot be {n}."); + } + /// Whether names a window function this engine can compute. A call with an /// OVER clause that this returns false for is a parse-level window function the engine has no evaluator /// for — then reports it by name. - public static bool IsWindowFunction(string name) => Registry.ContainsKey(name); + public static bool IsWindowFunction(string name) => + Registry.ContainsKey(name) || RunningAggregate.Supports(name.ToUpperInvariant()); public static WindowFunctionDef Lookup(string name) => - Registry.TryGetValue(name, out WindowFunctionDef? def) - ? def - : throw new NotSupportedException($"Window function '{name}' is not supported."); + Registry.TryGetValue(name, out WindowFunctionDef? def) ? def + : RunningAggregate.Supports(name.ToUpperInvariant()) ? Aggregate(name.ToUpperInvariant()) + : throw new NotSupportedException($"Window function '{name}' is not supported."); } diff --git a/src/LibRed/LibRed.Engine/Execution/WindowPartition.cs b/src/LibRed/LibRed.Engine/Execution/WindowPartition.cs new file mode 100644 index 000000000..925fd66a4 --- /dev/null +++ b/src/LibRed/LibRed.Engine/Execution/WindowPartition.cs @@ -0,0 +1,332 @@ +using System.Globalization; +using LibRed.Sql.Ast; + +namespace LibRed.Engine.Execution; + +/// +/// A window's frame clause with what it needs from each row: the bounds' offsets, as each row evaluates them, and — +/// for a RANGE offset — the one ORDER BY key and its direction. +/// +internal sealed record WindowFrameInput( + WindowFrame Frame, IReadOnlyList StartOffsets, IReadOnlyList EndOffsets, + IReadOnlyList? RangeKeys = null, bool Descending = false) +{ + /// The standard's default frame, which needs nothing from the rows. + public static readonly WindowFrameInput Default = new(WindowFrame.Default, [], []); +} + +/// How the call is written, beyond its name and arguments. +/// The argument is *, as in COUNT(*), which has no values. +/// The first argument is a Currency, which the statistical aggregates square exactly. +/// An aggregate over the distinct values in each frame. +/// IGNORE NULLS: rows whose first argument is Null are not counted. +/// FROM LAST: NTH_VALUE counts from the frame's last row. +/// An ordered-set aggregate's ordering: the directions of its last arguments, one each. +internal sealed record WindowCall( + bool Star = false, bool Currency = false, bool Distinct = false, bool IgnoreNulls = false, bool FromLast = false, + IReadOnlyList? WithinGroup = null) +{ + public static readonly WindowCall Plain = new(); +} + +/// +/// The rows of one frame, as positions in the partition's window order: [Start, End) less the excluded +/// [ExcludeStart, ExcludeEnd), which leaves in when it is set (EXCLUDE TIES). +/// +internal readonly record struct FrameRows(int Start, int End, int ExcludeStart = 0, int ExcludeEnd = 0, int Kept = -1) +{ + private int ExcludedFrom => Math.Clamp(ExcludeStart, Start, End); + + private int ExcludedTo => Math.Clamp(ExcludeEnd, ExcludedFrom, End); + + private bool KeepsOne => Kept >= ExcludedFrom && Kept < ExcludedTo; + + /// Whether a row is left out inside [Start, End). + public bool HasExclusion => ExcludedTo > ExcludedFrom; + + public int Count => End - Start - (ExcludedTo - ExcludedFrom) + (KeepsOne ? 1 : 0); + + /// The position of the frame's th row, from 0; -1 when it has no such row. + public int Nth(int n) + { + if (n < 0 || n >= Count) + return -1; + int before = ExcludedFrom - Start; + if (n < before) + return Start + n; + n -= before; + if (KeepsOne) + { + if (n == 0) + return Kept; + n--; + } + return ExcludedTo + n; + } + + /// The frame's rows, in window order. + public IEnumerable Positions() + { + for (int n = 0, count = Count; n < count; n++) + yield return Nth(n); + } + + /// The frame as at most three runs of adjacent positions, [From, To), in window order: the rows + /// before the exclusion, the one it keeps, and the rows after. + public IEnumerable<(int From, int To)> Runs() + { + if (ExcludedFrom > Start) + yield return (Start, ExcludedFrom); + if (KeepsOne) + yield return (Kept, Kept + 1); + if (End > ExcludedTo) + yield return (ExcludedTo, End); + } +} + +/// +/// One partition of a window's input, its rows already in window order, together with the peer-group +/// information the ranking functions need and the frame each row sees. A peer group is a run of adjacent rows +/// whose ORDER BY keys compare equal; with no ORDER BY the whole partition is one peer group, which is what makes +/// RANK constant over an unordered window. +/// +/// Positions are indexes into the partition's own window order, not into the input. +/// How the call is written; null for a plain one. +/// The frame clause; null for the default frame. +/// Which rows an aggregate's FILTER lets in; null when it has none. +internal sealed class WindowPartition( + IReadOnlyList peerStart, IReadOnlyList peerOrdinal, IReadOnlyList arguments, + WindowCall? call = null, WindowFrameInput? frame = null, IReadOnlyList? included = null) +{ + /// Whether an aggregate takes in the row at : its FILTER holds there. + public bool Includes(int position) => included is null || included[position]; + + private readonly WindowFrameInput _frame = frame ?? WindowFrameInput.Default; + + // Where each peer group starts, by ordinal; built on first use by GROUPS and the peer ends. + private int[]? _groupStarts; + + // The run of rows whose RANGE key is not Null — Null keys sort together at one end of the partition. + private (int Start, int End)? _keyed; + + // The positions whose first argument is not Null, ascending; built on first use by IGNORE NULLS. + private int[]? _valued; + + public WindowCall Call { get; } = call ?? WindowCall.Plain; + + /// Rows in this partition. + public int Count => peerStart.Count; + + /// Position of the first row of the peer group holding . RANK is this + /// plus one, which is why ranks skip after a tie. + public int PeerStart(int position) => peerStart[position]; + + /// Zero-based ordinal of the peer group holding , counted from the + /// start of the partition. DENSE_RANK is this plus one, which is why it does not skip. + public int PeerOrdinal(int position) => peerOrdinal[position]; + + /// The position after the last peer of the row at . + public int PeerEnd(int position) => GroupEnd(peerOrdinal[position]); + + /// The value of the call's th argument on the row at + /// — for the functions that take one (NTILE, LAG, FIRST_VALUE, …). + public object? Argument(int position, int argument) => arguments[position][argument]; + + /// How many arguments the call was given. + public int ArgumentCount => arguments.Count == 0 ? 0 : arguments[0].Length; + + /// + /// The position of the th row of , from 0, counted from its first row + /// or, , its last; -1 when it has no such row. Under IGNORE NULLS only the rows whose + /// first argument is not Null are counted. + /// + public int Pick(FrameRows frame, long n, bool fromLast) + { + var runs = frame.Runs(); + foreach ((int from, int to) in fromLast ? runs.Reverse() : runs) + { + int count = Call.IgnoreNulls ? ValuedBefore(to) - ValuedBefore(from) : to - from; + if (n < count) + { + int k = (int)(fromLast ? count - 1 - n : n); + return Call.IgnoreNulls ? Valued[ValuedBefore(from) + k] : from + k; + } + n -= count; + } + return -1; + } + + /// + /// The position rows before (or, , after) the one at + /// in the partition; -1 past its edge. Under IGNORE NULLS only the rows whose first + /// argument is not Null are counted, and offset 0 is still the row itself. + /// + public long Step(int position, long offset, bool forward) + { + if (!Call.IgnoreNulls || offset == 0) + { + long target = forward ? position + offset : position - offset; + return target >= 0 && target < Count ? target : -1; + } + long index = forward ? ValuedBefore(position + 1) + offset - 1 : ValuedBefore(position) - offset; + return index >= 0 && index < Valued.Length ? Valued[(int)index] : -1; + } + + private int[] Valued => _valued ??= Enumerable.Range(0, Count).Where(i => arguments[i][0] is not null).ToArray(); + + /// How many valued positions lie before . + private int ValuedBefore(int position) + { + int index = Array.BinarySearch(Valued, position); + return index >= 0 ? index : ~index; + } + + /// The rows of the frame the row at sees. + public FrameRows Frame(int position) + { + WindowFrame spec = _frame.Frame; + int start = Bound(spec.Start, position, _frame.StartOffsets, isStart: true); + int end = Math.Max(start, Bound(spec.End, position, _frame.EndOffsets, isStart: false)); + return spec.Exclusion switch + { + FrameExclusion.CurrentRow => new(start, end, position, position + 1), + FrameExclusion.Group => new(start, end, PeerStart(position), PeerEnd(position)), + FrameExclusion.Ties => new(start, end, PeerStart(position), PeerEnd(position), position), + _ => new(start, end), + }; + } + + private int[] GroupStarts => _groupStarts ??= Enumerable.Range(0, Count).Where(i => peerStart[i] == i).ToArray(); + + private int GroupEnd(int ordinal) => ordinal + 1 < GroupStarts.Length ? GroupStarts[ordinal + 1] : Count; + + /// + /// Where a bound puts the frame's first row () or the position after its last: in + /// ROWS the offset counts rows; in GROUPS, peer groups, the bound taking in all of its group; in RANGE, the ORDER + /// BY key's distance from the row's. CURRENT ROW is the row itself in ROWS, and its peer group otherwise. + /// + private int Bound(FrameBound bound, int position, IReadOnlyList offsets, bool isStart) + { + switch (bound.Kind) + { + case FrameBoundKind.UnboundedPreceding: + return 0; + case FrameBoundKind.UnboundedFollowing: + return Count; + case FrameBoundKind.CurrentRow: + return _frame.Frame.Unit == FrameUnit.Rows + ? (isStart ? position : position + 1) + : (isStart ? PeerStart(position) : PeerEnd(position)); + } + + bool preceding = bound.Kind == FrameBoundKind.Preceding; + object? offset = offsets[position]; + if (_frame.Frame.Unit == FrameUnit.Range) + return RangeBound(position, RangeOffset(offset), preceding, isStart); + + long n = WholeOffset(offset); + if (_frame.Frame.Unit == FrameUnit.Rows) + { + long row = preceding ? position - n : position + n; + return (int)Math.Clamp(isStart ? row : row + 1, 0, Count); + } + + long group = preceding ? PeerOrdinal(position) - n : PeerOrdinal(position) + n; + if (group < 0) + return 0; + if (group >= GroupStarts.Length) + return Count; + return isStart ? GroupStarts[group] : GroupEnd((int)group); + } + + /// + /// A RANGE offset bound: the first row (), or the row after the last, whose key lies + /// within of the current row's on the bound's side. A row with a Null key has only its + /// peers — the other Nulls — for an offset bound, and a Null key is never within an offset of a value. + /// + private int RangeBound(int position, object offset, bool preceding, bool isStart) + { + IReadOnlyList keys = _frame.RangeKeys!; + if (keys[position] is not { } key) + return isStart ? PeerStart(position) : PeerEnd(position); + + (int low, int high) = _keyed ??= Keyed(keys); + object here = RangeNumber(key); + + // Over the keyed run, a row's signed distance from this one grows along the window order, so the bound is + // where it first reaches the offset (a start) or first passes it (an end). + int lo = low, hi = high; + while (lo < hi) + { + int mid = (lo + hi) >>> 1; + int c = CompareDistance(RangeNumber(keys[mid]!), here, offset, preceding); + if (isStart ? c >= 0 : c > 0) + hi = mid; + else + lo = mid + 1; + } + return lo; + } + + /// The signed distance of from in window order, compared + /// with the bound's offset: negative for PRECEDING, positive for FOLLOWING. Exact where no side is a + /// floating-point number. + private int CompareDistance(object other, object here, object offset, bool preceding) + { + int direction = _frame.Descending ? -1 : 1; + if (other is not (double or float) && here is not (double or float) && offset is not (double or float)) + { + try + { + decimal distance = direction * (Convert.ToDecimal(other, CultureInfo.InvariantCulture) + - Convert.ToDecimal(here, CultureInfo.InvariantCulture)); + decimal limit = Convert.ToDecimal(offset, CultureInfo.InvariantCulture); + return distance.CompareTo(preceding ? -limit : limit); + } + catch (OverflowException) + { + // Past a Decimal: the Double comparison below. + } + } + double d = direction * (Convert.ToDouble(other, CultureInfo.InvariantCulture) + - Convert.ToDouble(here, CultureInfo.InvariantCulture)); + double l = Convert.ToDouble(offset, CultureInfo.InvariantCulture); + return d.CompareTo(preceding ? -l : l); + } + + private (int Start, int End) Keyed(IReadOnlyList keys) + { + int start = 0, end = Count; + while (start < end && keys[start] is null) + start++; + while (end > start && keys[end - 1] is null) + end--; + return (start, end); + } + + /// A RANGE key as a number: a date as its serial, as dates sort; text and other kinds are a type + /// mismatch. + private static object RangeNumber(object key) => key is string or char or Guid or byte[] + ? throw new InvalidCastException("Type mismatch: a RANGE offset needs a number or date to order by.") + : ExpressionEvaluator.ConversionNumber(key); + + /// A RANGE offset: a number of the key's units — days, for a date — Null and negative refused. + private static object RangeOffset(object? offset) + { + if (offset is null) + throw new ArgumentException("Invalid procedure call: a frame offset cannot be Null."); + object n = ExpressionEvaluator.ConversionNumber(offset); + return Convert.ToDouble(n, CultureInfo.InvariantCulture) >= 0 + ? n + : throw new ArgumentException($"Invalid procedure call: a frame offset cannot be {n}."); + } + + /// A ROWS or GROUPS offset: a whole number, read as CLng reads it, Null and negative refused. + private static long WholeOffset(object? offset) + { + if (offset is null) + throw new ArgumentException("Invalid procedure call: a frame offset cannot be Null."); + long n = Convert.ToInt64(ExpressionEvaluator.ConversionNumber(offset), CultureInfo.InvariantCulture); + return n >= 0 ? n : throw new ArgumentException($"Invalid procedure call: a frame offset cannot be {n}."); + } +} diff --git a/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs b/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs index 1813f952c..aa84f4b0e 100644 --- a/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs +++ b/src/LibRed/LibRed.Engine/Plan/PlanNodes.cs @@ -81,14 +81,17 @@ public sealed record HashJoinNode( /// /// Groups input rows by the key expressions and emits one row per /// group by evaluating — where aggregate calls are computed over -/// the group and other expressions see the group's key values. +/// the group and other expressions see the group's key values. are computed over the +/// groups HAVING keeps, each a row, and published to the projection and ORDER BY as +/// publishes them. /// public sealed record AggregateNode( PlanNode Input, IReadOnlyList GroupBy, IReadOnlyList Projection, Expression? Having, - IReadOnlyList OrderBy) : PlanNode + IReadOnlyList OrderBy, + IReadOnlyList? Windows = null) : PlanNode { public override IReadOnlyList Children => [Input]; } diff --git a/src/LibRed/LibRed.Engine/Planning/IndexSelection.cs b/src/LibRed/LibRed.Engine/Planning/IndexSelection.cs index c1deb3d47..2aff72399 100644 --- a/src/LibRed/LibRed.Engine/Planning/IndexSelection.cs +++ b/src/LibRed/LibRed.Engine/Planning/IndexSelection.cs @@ -1,4 +1,5 @@ using LibRed.Catalog; +using LibRed.Engine.Execution; using LibRed.Engine.Plan; using LibRed.Sql.Ast; @@ -89,6 +90,12 @@ private static PlanNode RewriteFilterOverScan(FilterNode filter, ScanNode scan, Expression? low = null, high = null; foreach (Expression conjunct in conjuncts) { + if (Between(conjunct, colName, alias, def) is { } range) + { + low ??= range.Low; + high ??= range.High; + continue; + } if (Bound(conjunct, colName, alias, def) is not { } b) continue; if (b.Op is BinaryOperator.GreaterThan or BinaryOperator.GreaterThanOrEqual) @@ -182,6 +189,33 @@ private static (BinaryOperator Op, Expression Value)? Bound(Expression conjunct, return null; } + /// If is col BETWEEN a AND b on column , the + /// two bounds lowest first; else null. + /// BETWEEN takes its bounds in either order, so which is the lower has to be known when planning. That + /// limits this to two literals of the same kind: numbers, texts or dates. A Null bound makes the whole test Null, + /// which a seek has no way to say, so it is left to the filter. + private static (Expression Low, Expression High)? Between(Expression conjunct, string colName, string alias, TableDef def) + { + if (conjunct is not BetweenExpression { Negated: false } between + || Column(between.Value, alias, def) is not { } column + || !string.Equals(column.Column, colName, StringComparison.OrdinalIgnoreCase) + || between.Low is not LiteralExpression { Value: { } low } + || between.High is not LiteralExpression { Value: { } high }) + return null; + + bool sameKind = (low, high) switch + { + (bool, _) or (_, bool) => false, + (string, string) or (DateTime, DateTime) => true, + _ => ExpressionEvaluator.IsNumeric(low) && ExpressionEvaluator.IsNumeric(high), + }; + if (!sameKind) + return null; + return ExpressionEvaluator.CompareForSort(low, high) <= 0 + ? (between.Low, between.High) + : (between.High, between.Low); + } + private static BinaryOperator Flip(BinaryOperator op) => op switch { BinaryOperator.GreaterThan => BinaryOperator.LessThan, @@ -385,11 +419,7 @@ JetDataType.Boolean or JetDataType.Byte or JetDataType.Int16 or JetDataType.Int3 ColumnReference { Table: { } t } => aliases.Contains(t), ColumnReference => false, // unqualified — can't attribute it to the outer side safely LiteralExpression or ParameterExpression or SystemVariableExpression => true, - BinaryExpression b => ReferencesOnly(b.Left, aliases) && ReferencesOnly(b.Right, aliases), - UnaryExpression u => ReferencesOnly(u.Operand, aliases), - FunctionCall f => f.Arguments.All(a => ReferencesOnly(a, aliases)), - InListExpression il => ReferencesOnly(il.Value, aliases) && il.Items.All(a => ReferencesOnly(a, aliases)), - _ => false, + _ => e.Operands()?.All(a => ReferencesOnly(a, aliases)) ?? false, }; /// The column reference if is a column of the given scan (its qualifier is @@ -411,11 +441,7 @@ JetDataType.Boolean or JetDataType.Byte or JetDataType.Int16 or JetDataType.Int3 // Same reasoning, and the same reason to be explicit: this switch's default means "no column reference" // — i.e. usable as a seek bound — so an unlisted node fails towards a WRONG seek, not a missed one. WindowFunction => true, - BinaryExpression b => HasColumnRef(b.Left) || HasColumnRef(b.Right), - UnaryExpression u => HasColumnRef(u.Operand), - FunctionCall f => f.Arguments.Any(HasColumnRef), - InListExpression il => HasColumnRef(il.Value) || il.Items.Any(HasColumnRef), - _ => false, // literals, parameters, system vars + _ => e.Operands()?.Any(HasColumnRef) ?? false, // no operands: literals, parameters, system vars }; /// The AND-conjuncts of a predicate; none at all for an absent one (a missing WHERE or HAVING). diff --git a/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs b/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs index 06bd1aa93..b2722c431 100644 --- a/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs +++ b/src/LibRed/LibRed.Engine/Planning/QueryPlanner.cs @@ -25,7 +25,7 @@ public PlanNode Plan(BoundStatement bound) // rows in the wrong order (measured against ACE). SetOperationStatement set => PageAndSort( new SetOperationNode(PlanStatement(set.Left), PlanStatement(set.Right), set.Operator), - set.OrderBy ?? [], set.Top, set.Offset), + OrderByPositions(set.OrderBy ?? [], _ => null), set.Top, set.Offset), ValuesStatement values => new ValuesNode(values.Rows), _ => throw new NotImplementedException( $"Planning for {statement.GetType().Name} is not yet implemented."), @@ -53,6 +53,18 @@ public static PlanNode PlanSelect(SelectStatement select) // applied over the source columns (before projection) so it can reference them. PlanNode node = PlanFrom(select.From); + // The sort runs on the source rows, below the projection, so a position becomes the projected expression + // it names; only a SELECT * sorts rows that are already its output. + if (select.OrderBy.Count > 0) + { + SelectStatement written = select; + select = select with + { + OrderBy = OrderByPositions(written.OrderBy, + written.IsSelectStar ? _ => null : position => ProjectedAt(written, position)), + }; + } + if (select.Where is not null) node = PushPredicates(node, select.Where); @@ -63,25 +75,21 @@ public static PlanNode PlanSelect(SelectStatement select) var windows = new List(); select = ExtractWindows(select, windows); + // A window whose arguments or keys hold an aggregate — RANK() OVER (ORDER BY SUM(x)) — makes the query + // grouped by itself, as an aggregate in the projection does. bool aggregate = select.GroupBy.Count > 0 || select.Having is not null - || select.Projection.Any(i => HasAggregate(i.Value)); + || select.Projection.Any(i => HasAggregate(i.Value)) + || windows.Any(w => w.Function.Expressions().Any(HasAggregate)); - if (windows.Count > 0) - { - if (aggregate) - // AggregateNode owns the projection, HAVING and ORDER BY and collapses rows, so a window over - // grouped output would need its projection split across the two nodes. EF Core always puts such - // a window in its own derived table, so nothing needs this yet — refuse loudly rather than hand - // the call to AggregateNode, whose per-group evaluation swallows the resulting error. - throw new NotSupportedException( - "A window function over a grouped query (GROUP BY / HAVING / an aggregate projection) is not supported."); + // Over a grouped query the windows run over the groups, after HAVING and before the projection and ORDER BY + // — the standard's order — so the aggregate node computes them itself. + if (windows.Count > 0 && !aggregate) node = new WindowNode(node, windows); - } if (aggregate) // The aggregate node owns ORDER BY: its keys are evaluated in the group scope (so they can // reference grouping expressions / aggregates), not over the already-projected output. - node = new AggregateNode(node, select.GroupBy, select.Projection, select.Having, select.OrderBy); + node = new AggregateNode(node, select.GroupBy, select.Projection, select.Having, select.OrderBy, windows); else if (select.OrderBy.Count > 0) node = PushSort(node, select.OrderBy); @@ -119,6 +127,39 @@ public static PlanNode PlanSelect(SelectStatement select) return node; } + /// + /// ORDER BY items that are a whole number written as such name the output column at that position (verified vs + /// ACE, as SQL-92 has it: ORDER BY 2 DESC sorts by the second column, and so does (2)); any other constant — + /// 1.5, '2', 1 + 1 — is left as a constant. A position below 1 names nothing and is an error, as is one past the + /// last column. gives the expression at a position, or null where the rows sorted + /// are the output itself and the position is read from them. + /// + private static IReadOnlyList OrderByPositions( + IReadOnlyList orderBy, Func projected) => + orderBy.Select(item => item.Value is LiteralExpression { Value: int or long or short or byte } literal + ? item with + { + Value = Convert.ToInt32(literal.Value, System.Globalization.CultureInfo.InvariantCulture) is var position + && position >= 1 + ? projected(position) ?? new OutputColumnPosition(position) + : throw NoSuchPosition(literal.Value!), + } + : item).ToList(); + + private static InvalidOperationException NoSuchPosition(object position) => + new($"'{position}' is not a valid field name or expression: ORDER BY {position} names no output column."); + + /// The projected expression at a 1-based position. A table.* before it would need the table's + /// columns counted, which the planner cannot, and is refused. + private static Expression ProjectedAt(SelectStatement select, int position) + { + if (position > select.Projection.Count) + throw NoSuchPosition(position); + if (select.Projection.Take(position).Any(item => item.Value is StarExpression or QualifiedStarExpression)) + throw new NotSupportedException($"ORDER BY {position} after a table.* in the projection is not supported."); + return select.Projection[position - 1].Value; + } + /// /// Places an ORDER BY as deep in the join tree as it can go: when every key comes from one side of a join, /// sorting that side and letting the join stream produces the same order as sorting the join's output — @@ -175,24 +216,23 @@ private static PlanNode PushSort(PlanNode node, IReadOnlyList keys) /// The aggregate function names recognised by the planner/executor. Includes the Access statistical /// aggregates StDev/StDevP (sample/population standard deviation) and Var/VarP (sample/population variance); - /// the "StdDev"/"StdDevP" spellings are accepted as aliases. + /// the "StdDev"/"StdDevP" spellings are accepted as aliases. Beyond Access, a LibRed extension: the standard's + /// names for the statistics, its binary set functions (CORR, COVAR_*, REGR_*) — all of them + /// 's — and its ordered-set aggregates PERCENTILE_CONT, PERCENTILE_DISC + /// and LISTAGG. internal static bool IsAggregate(string name) => - name.ToUpperInvariant() is "COUNT" or "SUM" or "AVG" or "MIN" or "MAX" or "FIRST" or "LAST" - or "STDEV" or "STDEVP" or "STDDEV" or "STDDEVP" or "VAR" or "VARP"; + name.ToUpperInvariant() is var upper + && (upper is "FIRST" or "LAST" or "PERCENTILE_CONT" or "PERCENTILE_DISC" or "LISTAGG" + || Execution.RunningAggregate.Supports(upper)); internal static bool HasAggregate(Expression e) => e switch { FunctionCall f when IsAggregate(f.Name) => true, - FunctionCall f => f.Arguments.Any(HasAggregate), - BinaryExpression b => HasAggregate(b.Left) || HasAggregate(b.Right), - UnaryExpression u => HasAggregate(u.Operand), - // An aggregate inside a CASE has to be found here so it is computed per group and handed to the - // evaluator, rather than being reached during evaluation when no group scope can resolve it. The - // standard says the same: aggregates in a WHEN are evaluated before the CASE, not by it. Conditions - // count as well as results — HAVING CASE WHEN COUNT(*) > 1 … puts the aggregate in the condition. - CaseExpression c => c.WhenClauses.Any(w => HasAggregate(w.Condition) || HasAggregate(w.Result)) - || (c.ElseResult is not null && HasAggregate(c.ElseResult)), - _ => false, + // Operands include a CASE's: an aggregate inside a CASE has to be found here so it is computed per group + // and handed to the evaluator, rather than being reached during evaluation when no group scope can + // resolve it. The standard says the same: aggregates in a WHEN are evaluated before the CASE, not by it. + // Conditions count as well as results — HAVING CASE WHEN COUNT(*) > 1 … puts the aggregate in the condition. + _ => e.Operands()?.Any(HasAggregate) ?? false, }; /// @@ -222,48 +262,19 @@ private static SelectStatement ExtractWindows(SelectStatement select, List e switch { WindowFunction => true, - FunctionCall f => f.Arguments.Any(HasWindow), - BinaryExpression b => HasWindow(b.Left) || HasWindow(b.Right), - UnaryExpression u => HasWindow(u.Operand), - CaseExpression c => c.WhenClauses.Any(w => HasWindow(w.Condition) || HasWindow(w.Result)) - || (c.ElseResult is not null && HasWindow(c.ElseResult)), - InListExpression il => HasWindow(il.Value) || il.Items.Any(HasWindow), - _ => false, + _ => e.Operands()?.Any(HasWindow) ?? false, }; private static Expression LiftWindows(Expression e, List windows) { - switch (e) - { - case WindowFunction w: - // A name no identifier can spell: IDENTIFIER allows '$' only as a trailing character, so this - // cannot collide with a real column and be silently shadowed. - string name = $"$window{windows.Count}"; - windows.Add(new WindowOutput(name, w)); - return new ColumnReference(null, name); - case FunctionCall f: - return f with { Arguments = f.Arguments.Select(a => LiftWindows(a, windows)).ToList() }; - case BinaryExpression b: - return b with { Left = LiftWindows(b.Left, windows), Right = LiftWindows(b.Right, windows) }; - case UnaryExpression u: - return u with { Operand = LiftWindows(u.Operand, windows) }; - case CaseExpression c: - return c with - { - WhenClauses = c.WhenClauses - .Select(w => w with { Condition = LiftWindows(w.Condition, windows), Result = LiftWindows(w.Result, windows) }) - .ToList(), - ElseResult = c.ElseResult is null ? null : LiftWindows(c.ElseResult, windows), - }; - case InListExpression il: - return il with - { - Value = LiftWindows(il.Value, windows), - Items = il.Items.Select(i => LiftWindows(i, windows)).ToList(), - }; - default: - return e; - } + if (e is not WindowFunction w) + return e.MapOperands(o => LiftWindows(o, windows)); + + // A name no identifier can spell: IDENTIFIER allows '$' only as a trailing character, so this cannot + // collide with a real column and be silently shadowed. + string name = $"$window{windows.Count}"; + windows.Add(new WindowOutput(name, w)); + return new ColumnReference(null, name); } private static PlanNode PlanFrom(TableReference? from) => from switch @@ -382,11 +393,8 @@ private static (PlanNode Node, List Unplaced) Place( ColumnReference { Table: { } t } => Add(acc, t), ColumnReference => false, // unqualified — can't determine its table LiteralExpression or ParameterExpression or SystemVariableExpression => true, - BinaryExpression b => Collect(b.Left, acc) && Collect(b.Right, acc), - UnaryExpression u => Collect(u.Operand, acc), - FunctionCall f => f.Arguments.All(a => Collect(a, acc)), - InListExpression il => Collect(il.Value, acc) && il.Items.All(a => Collect(a, acc)), - _ => false, // subqueries (scalar/EXISTS/IN), qualified star, etc. — don't push + // Anything without operands — subqueries (scalar/EXISTS/IN), qualified star, etc. — isn't pushed. + _ => e.Operands()?.All(a => Collect(a, acc)) ?? false, }; static bool Add(HashSet acc, string t) { acc.Add(t); return true; } } diff --git a/src/LibRed/LibRed.Engine/Planning/ViewExpander.cs b/src/LibRed/LibRed.Engine/Planning/ViewExpander.cs index b8f5f101d..328a9b0bc 100644 --- a/src/LibRed/LibRed.Engine/Planning/ViewExpander.cs +++ b/src/LibRed/LibRed.Engine/Planning/ViewExpander.cs @@ -62,14 +62,7 @@ private static Expression RewriteExpression( Value = RewriteExpression(i.Value, views, parser, active), Query = Rewrite(i.Query, views, parser, active), }, - BinaryExpression b => b with - { - Left = RewriteExpression(b.Left, views, parser, active), - Right = RewriteExpression(b.Right, views, parser, active), - }, - UnaryExpression u => u with { Operand = RewriteExpression(u.Operand, views, parser, active) }, - FunctionCall f => f with { Arguments = f.Arguments.Select(a => RewriteExpression(a, views, parser, active)).ToList() }, - _ => expr, + _ => expr.MapOperands(o => RewriteExpression(o, views, parser, active)), }; private static TableReference RewriteSource( diff --git a/src/LibRed/LibRed.Sql/Ast/ExpressionTree.cs b/src/LibRed/LibRed.Sql/Ast/ExpressionTree.cs new file mode 100644 index 000000000..d9d72a79f --- /dev/null +++ b/src/LibRed/LibRed.Sql/Ast/ExpressionTree.cs @@ -0,0 +1,43 @@ +namespace LibRed.Sql.Ast; + +/// +/// The expressions an expression is built from, so that a walker names only the nodes it treats specially and a new +/// kind of node is taught to every walker here, once. +/// +public static class ExpressionTree +{ + /// + /// The expressions directly inside and evaluated in its scope: operands, function + /// arguments (and an aggregate's FILTER), IN-list items, BETWEEN bounds, and a CASE's conditions and results. Null for anything else — a leaf, + /// a subquery, an IN subquery or a window function — which each walker treats in its own way. + /// + public static IEnumerable? Operands(this Expression expression) => expression switch + { + BinaryExpression b => new[] { b.Left, b.Right }, + UnaryExpression u => new[] { u.Operand }, + FunctionCall f => f.Filter is null ? f.Arguments : f.Arguments.Append(f.Filter), + InListExpression il => il.Items.Prepend(il.Value), + BetweenExpression be => new[] { be.Value, be.Low, be.High }, + CaseExpression c => c.WhenClauses + .SelectMany(w => new[] { w.Condition, w.Result }) + .Concat(c.ElseResult is { } otherwise ? new[] { otherwise } : []), + _ => null, + }; + + /// with applied to each of its + /// , in order; the expression itself when it has none. + public static Expression MapOperands(this Expression expression, Func map) => expression switch + { + BinaryExpression b => b with { Left = map(b.Left), Right = map(b.Right) }, + UnaryExpression u => u with { Operand = map(u.Operand) }, + FunctionCall f => f with { Arguments = f.Arguments.Select(map).ToList(), Filter = f.Filter is null ? null : map(f.Filter) }, + InListExpression il => il with { Value = map(il.Value), Items = il.Items.Select(map).ToList() }, + BetweenExpression be => be with { Value = map(be.Value), Low = map(be.Low), High = map(be.High) }, + CaseExpression c => c with + { + WhenClauses = c.WhenClauses.Select(w => w with { Condition = map(w.Condition), Result = map(w.Result) }).ToList(), + ElseResult = c.ElseResult is null ? null : map(c.ElseResult), + }, + _ => expression, + }; +} diff --git a/src/LibRed/LibRed.Sql/Ast/Expressions.cs b/src/LibRed/LibRed.Sql/Ast/Expressions.cs index df1b4c78b..f763cb258 100644 --- a/src/LibRed/LibRed.Sql/Ast/Expressions.cs +++ b/src/LibRed/LibRed.Sql/Ast/Expressions.cs @@ -3,8 +3,10 @@ namespace LibRed.Sql.Ast; /// Base type for scalar/boolean expressions. public abstract record Expression : SqlNode; -/// A literal constant (number, string, date, boolean or null). -public sealed record LiteralExpression(object? Value) : Expression; +/// A literal constant (number, string, date, boolean or null). is the exact value of +/// a number written with a decimal point and no exponent, its scale the digits written after the point (1.50 is +/// 1.50); ACE reads such a literal as a Decimal and keeps no more places than that in a product or quotient. +public sealed record LiteralExpression(object? Value, decimal? Written = null) : Expression; /// A reference to a column, optionally table-qualified. public sealed record ColumnReference(string? Table, string Column) : Expression; @@ -18,6 +20,11 @@ public sealed record ParameterExpression(string Name) : Expression; /// without the leading @@. public sealed record SystemVariableExpression(string Name) : Expression; +/// The th (1-based) column of the rows being sorted — what ORDER BY n names +/// where those rows are already the query's output (a SELECT *, or a set operation). Made by the planner, never +/// parsed. +public sealed record OutputColumnPosition(int Position) : Expression; + /// * in a projection or aggregate. public sealed record StarExpression : Expression; @@ -29,7 +36,7 @@ public enum BinaryOperator { Add, Subtract, Multiply, Divide, Modulo, IntDivide, Power, Concat, Equal, NotEqual, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual, - And, Or, Like, In, + And, Or, Xor, Eqv, Imp, Like, In, BitAnd, BitOr, BitXor, // Access bitwise operators BAND / BOR / BXOR (integers only) } @@ -41,16 +48,59 @@ public sealed record UnaryExpression(UnaryOperator Operator, Expression Operand) /// A scalar/aggregate function call, e.g. Count(*), IIf(...), Format(...). /// is set for the ANSI aggregate form COUNT(DISTINCT col) — the aggregate -/// runs over the distinct set of the argument's values (not distinct rows). -public sealed record FunctionCall(string Name, IReadOnlyList Arguments, bool Distinct = false) : Expression; +/// runs over the distinct set of the argument's values (not distinct rows). +/// is set for an ordered-set aggregate, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY +/// x DESC): its ORDER BY keys are the last of the , so that every walker sees them as +/// it sees an argument, and this holds their directions, one per key. is an aggregate's +/// FILTER (WHERE …): only the rows it is true for go in. +public sealed record FunctionCall( + string Name, IReadOnlyList Arguments, bool Distinct = false, + IReadOnlyList? WithinGroup = null, Expression? Filter = null) : Expression +{ + /// Whether is an ordered-set aggregate, which takes WITHIN GROUP and needs it. + public static bool IsOrderedSetAggregate(string name) => + name.Equals("PERCENTILE_CONT", StringComparison.OrdinalIgnoreCase) + || name.Equals("PERCENTILE_DISC", StringComparison.OrdinalIgnoreCase) + || name.Equals("LISTAGG", StringComparison.OrdinalIgnoreCase); + + /// The WITHIN GROUP keys: the last arguments, one per direction. + public IReadOnlyList WithinGroupKeys => + WithinGroup is null ? [] : Arguments.Skip(Arguments.Count - WithinGroup.Count).ToList(); +} /// The OVER (…) of a window function: how the input is cut into partitions and how rows are /// ordered within one. An empty means a single partition over the whole input; -/// an empty means every row of a partition is a peer. A frame clause belongs here -/// when one is needed — adding it is a new optional property on this record and nothing else. +/// an empty means every row of a partition is a peer. A null +/// is the standard's default frame, RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +/// (). public sealed record WindowSpec( IReadOnlyList PartitionBy, - IReadOnlyList OrderBy) : SqlNode; + IReadOnlyList OrderBy, + WindowFrame? Frame = null) : SqlNode; + +/// How a frame's bounds are counted: in rows, in ORDER BY values, or in peer groups. +public enum FrameUnit { Rows, Range, Groups } + +/// The kinds of frame bound, in window order — a frame's start never comes after its end in this order. +public enum FrameBoundKind { UnboundedPreceding, Preceding, CurrentRow, Following, UnboundedFollowing } + +/// One end of a window frame; is set for n PRECEDING and n FOLLOWING. +public sealed record FrameBound(FrameBoundKind Kind, Expression? Offset = null) : SqlNode; + +/// The rows a frame leaves out around the current row: none, the row itself, its peer group, or its peers +/// but not itself. +public enum FrameExclusion { NoOthers, CurrentRow, Group, Ties } + +/// A window's frame: the rows of the partition, around the current row, that a frame-reading function — +/// an aggregate, FIRST_VALUE, … — sees. +public sealed record WindowFrame(FrameUnit Unit, FrameBound Start, FrameBound End, FrameExclusion Exclusion = FrameExclusion.NoOthers) + : SqlNode +{ + /// RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: with an ORDER BY, the partition up to the + /// current row's last peer; without one, the whole partition, every row then being a peer. + public static readonly WindowFrame Default = + new(FrameUnit.Range, new(FrameBoundKind.UnboundedPreceding), new(FrameBoundKind.CurrentRow)); +} /// /// A window function call: ROW_NUMBER() OVER (PARTITION BY … ORDER BY …). Access has none of these — @@ -63,10 +113,30 @@ public sealed record WindowSpec( /// AggregateNode. As a sibling record it falls through to "not an aggregate", which is correct — a window /// function returns one value per ROW, not per group, whatever its name. /// +/// A windowed aggregate over the distinct values of its argument in each frame. +/// IGNORE NULLS (true) or RESPECT NULLS (false); null when neither is written. +/// FROM LAST (true) or FROM FIRST (false); null when neither is written. +/// An ordered-set aggregate's ordering, as on . +/// An aggregate's FILTER (WHERE …), as on . public sealed record WindowFunction( string Name, IReadOnlyList Arguments, - WindowSpec Over) : Expression; + WindowSpec Over, + bool Distinct = false, + bool? IgnoreNulls = null, + bool? FromLast = null, + IReadOnlyList? WithinGroup = null, + Expression? Filter = null) : Expression +{ + /// Every expression the call evaluates on a row: arguments, FILTER, partition and sort keys, and frame + /// offsets. + public IEnumerable Expressions() => + Arguments + .Concat(Filter is null ? [] : [Filter]) + .Concat(Over.PartitionBy) + .Concat(Over.OrderBy.Select(o => o.Value)) + .Concat(new[] { Over.Frame?.Start.Offset, Over.Frame?.End.Offset }.OfType()); +} /// A subquery used as a scalar value: (SELECT … ). May correlate to the outer query. /// @@ -93,6 +163,10 @@ public sealed record InSubqueryExpression(Expression Value, SqlStatement Query, /// three-valued semantics as . public sealed record InListExpression(Expression Value, IReadOnlyList Items, bool Negated) : Expression; +/// value [NOT] BETWEEN low AND high: inclusive, with the bounds in either order, and Null when any of +/// the three is Null. +public sealed record BetweenExpression(Expression Value, Expression Low, Expression High, bool Negated) : Expression; + /// /// The DEFAULT keyword used as a row value in an INSERT's table value constructor: /// VALUES ('Advertisement', DEFAULT). It is a marker rather than a value — the column takes its diff --git a/src/LibRed/LibRed.Sql/Ast/Statements.cs b/src/LibRed/LibRed.Sql/Ast/Statements.cs index 3c2ea16d5..89f7137e6 100644 --- a/src/LibRed/LibRed.Sql/Ast/Statements.cs +++ b/src/LibRed/LibRed.Sql/Ast/Statements.cs @@ -135,7 +135,15 @@ public sealed record ColumnDefinition( /// A calculated column's expression, from the LibRed-only AS (…) clause, kept as source /// text because that text is what goes on disk and what the engine re-evaluates. Null for an ordinary /// column. - string? Calculated = null); + string? Calculated = null, + /// A trailing IDENTITY [(seed [, increment])] attribute, or null when the column has none. + /// The last one written wins when several are. + IdentityAttribute? Identity = null); + +/// ACE's IDENTITY [(seed [, increment])] column attribute. It makes a Long column an AutoNumber +/// counting from its seed by its increment — each 1 when omitted, replacing any the type declared — and is +/// ignored on every other type. +public sealed record IdentityAttribute(int? Seed, int? Increment); /// Referential action for a foreign key's ON DELETE / ON UPDATE clause. Jet records only /// enforce + cascade-update + cascade-delete, so NoAction/SetNull/SetDefault collapse to "no cascade". @@ -143,7 +151,8 @@ public enum ReferentialAction { NoAction, Cascade, SetNull, SetDefault } /// A FOREIGN KEY constraint (table-level, or a column-level REFERENCES): the child columns, /// the referenced (parent) table and its columns, the ON DELETE / ON UPDATE actions, and whether the -/// FOREIGN KEY NO INDEX modifier was given (suppresses the backing index). +/// FOREIGN KEY NO INDEX modifier was given (suppresses the backing index). An empty +/// means REFERENCES table with no column list: the parent's primary key. public sealed record ForeignKeyConstraint( string? Name, IReadOnlyList Columns, @@ -254,8 +263,14 @@ public sealed record CreateActionProcedureStatement( /// One action of an ALTER TABLE statement (Access allows exactly one per statement). public abstract record AlterTableAction; -/// ADD [COLUMN] field type … — add a new column (with its inline constraints). -public sealed record AddColumnAction(ColumnDefinition Column) : AlterTableAction; +/// ADD [COLUMN] field type … — add a new column with its inline constraints: a PRIMARY KEY +/// (, named when a CONSTRAINT name was given), +/// a constraint and a column-level , each null when absent. +public sealed record AddColumnAction( + ColumnDefinition Column, + ForeignKeyConstraint? References = null, + UniqueConstraint? Unique = null, + string? PrimaryKeyName = null) : AlterTableAction; /// ADD CONSTRAINT … FOREIGN KEY … — add a foreign key. public sealed record AddForeignKeyAction(ForeignKeyConstraint ForeignKey) : AlterTableAction; @@ -272,7 +287,8 @@ public sealed record AddCheckAction(CheckConstraint Check) : AlterTableAction; /// ALTER COLUMN field type[(size[,scale])] [NOT NULL|NULL] [DEFAULT expr] — change a column's data /// type, and optionally its nullability (: true = NOT NULL, false = NULL, null = leave /// as-is) and default. -public sealed record AlterColumnAction(string Field, string TypeName, int? Size, int? Scale, string? Default = null, bool? NotNull = null) : AlterTableAction; +public sealed record AlterColumnAction(string Field, string TypeName, int? Size, int? Scale, string? Default = null, bool? NotNull = null, + IdentityAttribute? Identity = null) : AlterTableAction; /// ALTER COLUMN field SET DEFAULT expr — set (replace) a column's default, without retyping it. public sealed record AlterColumnSetDefaultAction(string Field, string Default) : AlterTableAction; diff --git a/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 b/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 index 377754bb1..509fe49a4 100644 --- a/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 +++ b/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 @@ -4,12 +4,36 @@ // derived-table subqueries, WHERE, ORDER BY, TOP. The parse tree is lowered into // LibRed.Sql.Ast by AstBuilder, so the rest of the engine never sees these generated types. // -// Dialect notes (vs ANSI): '&' string concat; MOD / '\' operators; '*'/'?' LIKE wildcards; -// TOP n (no OFFSET); #1/1/2020# date literals; [bracketed] and `backtick` identifiers; -// booleans -1/0. +// Dialect notes (vs ANSI): '&' string concat; MOD / '\' operators; TOP n (no OFFSET); +// #1/1/2020# date literals; [bracketed] and `backtick` identifiers; booleans -1/0. grammar AccessSql; +@lexer::members { + // Whether the input ahead is any number of dashes and then a number: a digit, or a point and a digit. + private bool DashesThenNumber() + { + int i = 1; + while (InputStream.LA(i) == '-') + i++; + int c = InputStream.LA(i); + return c is >= '0' and <= '9' || c == '.' && InputStream.LA(i + 1) is >= '0' and <= '9'; + } +} + +@parser::members { + // The precedence the generated parser gives a BETWEEN's upper bound: that of the comparisons, one above BETWEEN. + private const int BetweenBoundPrecedence = 13; + + // ANTLR parses the middle operand of an alternative at precedence 0 and has no syntax to raise it, which lets a + // BETWEEN's lower bound swallow the AND that ends it: `x BETWEEN 1 AND 10 AND y = 2` would read as + // `x BETWEEN (1 AND 10) AND (y = 2)`. The lower bound is the only operand parsed while its BETWEEN has no lower + // bound yet, so it is given the upper bound's precedence here. + public override void EnterRecursionRule(ParserRuleContext localctx, int state, int ruleIndex, int precedence) => + base.EnterRecursionRule(localctx, state, ruleIndex, + localctx.Parent is BetweenExprContext { lo: null } ? Math.Max(precedence, BetweenBoundPrecedence) : precedence); +} + // A single statement, optionally terminated by ';' (EF Core emits a trailing semicolon). statement : parametersClause? (ifThenStatement | createTableStatement | createIndexStatement | createViewStatement | createProcedureStatement | alterTableStatement | dropStatement | insertStatement | updateStatement | deleteStatement | transactionStatement | executeStatement | systemVariableSelect | queryExpression) SEMI? EOF ; @@ -25,14 +49,16 @@ executeStatement : (EXECUTE | EXEC) name=identifier (expression (COMMA expressio // UPDATE table SET col = expr, … [WHERE criteria]. The WHERE criteria is an ordinary expression, the same // as a SELECT's; each SET value expression may reference the row's current column values. // UPDATE tableexpression SET col=expr, … [WHERE …]. The tableexpression is a table SOURCE (Access allows a -// join here), and a SET target may be table-qualified (col or alias.col) to touch a specific joined table. -updateStatement : UPDATE tableSource SET assignment (COMMA assignment)* whereClause? ; +// join here) or, as in a FROM clause, a comma list of them (verified vs ACE: UPDATE a, b SET … is accepted), and a +// SET target may be table-qualified (col or alias.col) to touch a specific joined table. +updateStatement : UPDATE tableSource (COMMA tableSource)* SET assignment (COMMA assignment)* whereClause? ; assignment : target=columnRef EQ expression ; // DELETE [table.* | *] FROM tableexpression [WHERE …]. For a join, the `table.*` target selects which // table's rows to delete; a bare `*` (or no target) is only valid for a single table — a join without a // `table.*` target is ambiguous and rejected at execution (matching Access, which asks you to specify it). -deleteStatement : DELETE (target=identifier DOT STAR | STAR)? FROM tableSource whereClause? ; +// As in a FROM clause, the tableexpression may be a comma list of sources (verified vs ACE). +deleteStatement : DELETE (target=identifier DOT STAR | STAR)? FROM tableSource (COMMA tableSource)* whereClause? ; // A FROM-less SELECT of system variables only — ACE allows `SELECT @@IDENTITY` / `SELECT @@ROWCOUNT` // (and a comma list of them) with no FROM clause. Listed before queryExpression so it is preferred; a @@ -139,31 +165,41 @@ calculatedClause : AS LPAREN expression RPAREN ; // A second word handles two-word ANSI aliases like CHARACTER VARYING / BIT VARYING. // Up to three words to cover multi-word SQL type names: "char varying", "national character varying", etc. -dataType : typeName=identifier extra=identifier? extra2=identifier? (LPAREN size=signedInteger (COMMA scale=signedInteger)? RPAREN)? ; +// IDENTITY is reserved, as ACE reserves it, so it is named here as a type of its own: `Id IDENTITY(5, 2)`. After a +// type it is a column constraint instead (see IdentityConstraint). +dataType : (typeName=identifier | identityType=IDENTITY) extra=identifier? extra2=identifier? (LPAREN size=signedInteger (COMMA scale=signedInteger)? RPAREN)? ; // A possibly-negative integer — needed for a descending COUNTER(seed, increment) whose increment is negative. signedInteger : MINUS? INTEGER_LITERAL ; // Single-field constraints (after the column's data type). A CONSTRAINT name may prefix any of them. +// PRIMARY KEY and UNIQUE take an optional CLUSTERED or NONCLUSTERED, in a column or table constraint alike, as +// ACE's CONSTRAINT clause does. ACE stores nothing for either word — the file is byte-identical without it and +// DAO reports Clustered = False — so it is parsed and dropped. ACE rejects it anywhere else: after FOREIGN KEY, +// between PRIMARY and KEY, on a bare column, or in CREATE INDEX. columnConstraint : NOT NULL # NotNullConstraint | NULL # NullableConstraint | DEFAULT expression # DefaultConstraint | WITH (COMPRESSION | COMP) # CompressionConstraint | (CONSTRAINT cname=identifier)? CHECK LPAREN checkBody RPAREN # CheckColumnConstraint - | (CONSTRAINT cname=identifier)? PRIMARY KEY # PrimaryKeyConstraint - | (CONSTRAINT cname=identifier)? UNIQUE # UniqueColumnConstraint + | (CONSTRAINT cname=identifier)? PRIMARY KEY clusteredOption? # PrimaryKeyConstraint + | (CONSTRAINT cname=identifier)? UNIQUE clusteredOption? # UniqueColumnConstraint | (CONSTRAINT cname=identifier)? REFERENCES refTable=identifier (LPAREN refColumns+=identifier (COMMA refColumns+=identifier)* RPAREN)? foreignKeyAction* # ColumnReferencesConstraint + // IDENTITY [(seed [, increment])] — ACE's AutoNumber attribute. It may follow only the type, NULL/NOT NULL or + // another IDENTITY, which the AST builder checks; it makes a Long column an AutoNumber and is ignored on any + // other type, both as ACE does. + | IDENTITY (LPAREN seed=signedInteger (COMMA increment=signedInteger)? RPAREN)? # IdentityConstraint ; // EF Core emits named table constraints: CONSTRAINT `PK_x` PRIMARY KEY (`col`, ...) and // CONSTRAINT `FK_x` FOREIGN KEY (`col`, ...) REFERENCES `Parent` (`col`, ...) ON DELETE CASCADE. tableConstraint - : (CONSTRAINT name=identifier)? PRIMARY KEY + : (CONSTRAINT name=identifier)? PRIMARY KEY clusteredOption? LPAREN columns+=identifier (COMMA columns+=identifier)* RPAREN # PrimaryKeyTableConstraint - | (CONSTRAINT name=identifier)? UNIQUE + | (CONSTRAINT name=identifier)? UNIQUE clusteredOption? LPAREN columns+=identifier (COMMA columns+=identifier)* RPAREN # UniqueTableConstraint | (CONSTRAINT name=identifier)? FOREIGN KEY (noIndex=NO INDEX)? LPAREN columns+=identifier (COMMA columns+=identifier)* RPAREN @@ -364,21 +400,39 @@ orderByClause : ORDER BY orderByItem (COMMA orderByItem)* ; orderByItem : expression (dir=(ASC | DESC))? ; expression - : NOT expression # NotExpr - | BNOT expression # BitNotExpr - | MINUS expression # NegateExpr - | left=expression CARET right=expression # PowExpr - | left=expression op=(STAR | SLASH | MOD | BACKSLASH) right=expression # MulDivExpr - | left=expression op=(PLUS | MINUS | AMP) right=expression # AddConcatExpr + : left=expression CARET right=expression # PowExpr + // Negation binds looser than '^' and tighter than '*' (VBA operator precedence), except that a minus written + // against a number is part of that number (see AstBuilder.SignedNumber). A unary plus sits with it and leaves its + // operand as it is, text included (verified vs ACE: +'abc' is 'abc'). + | op=(MINUS | PLUS) expression # NegateExpr + // '*' '/', then '\', then MOD, each its own level (VBA operator precedence; verified vs ACE: 7 \ 2 * 3 is 1, + // 5 MOD 3 * 2 is 5, 10 MOD 4 \ 2 is 0). + | left=expression op=(STAR | SLASH) right=expression # MulDivExpr + | left=expression op=BACKSLASH right=expression # IntDivExpr + | left=expression op=MOD right=expression # ModExpr + | left=expression op=(PLUS | MINUS) right=expression # AddSubExpr + // '&' binds looser than '+'/'-' and tighter than the comparisons (VBA operator precedence; verified vs + // ACE: 1 & 2 + 3 is '15'). + | left=expression op=AMP right=expression # ConcatExpr | left=expression op=(EQ | NEQ | LT | LTE | GT | GTE) right=expression # ComparisonExpr + // The lower bound is parsed at the upper bound's precedence (see EnterRecursionRule above), so the AND between + // them is never taken into it. | val=expression not=NOT? BETWEEN lo=expression AND hi=expression # BetweenExpr | left=expression not=NOT? LIKE right=expression # LikeExpr | val=expression not=NOT? IN LPAREN sub=queryExpression RPAREN # InSubqueryExpr | val=expression not=NOT? IN LPAREN items+=expression (COMMA items+=expression)* RPAREN # InExpr | operand=expression IS not=NOT? NULL # IsNullExpr - | left=expression op=(BAND | BOR | BXOR) right=expression # BitwiseExpr - | left=expression AND right=expression # AndExpr - | left=expression OR right=expression # OrExpr + // NOT binds looser than the comparisons and tighter than AND (VBA operator precedence; verified vs ACE: + // NOT 1 = 2 is True). BNOT sits with it, and each bitwise operator with its logical one, left to right + // (verified vs ACE: BNOT 1 + 1 is -3, NOT 0 BAND 1 is 1, 2 AND 1 BAND 3 is 3, 0 OR 0 BOR 4 is 4). + | op=(NOT | BNOT) expression # NotExpr + | left=expression op=(AND | BAND) right=expression # AndExpr + | left=expression op=(OR | BOR) right=expression # OrExpr + // XOR, then EQV, then IMP, each looser than the one before (VBA operator precedence; verified vs ACE: + // TRUE XOR TRUE OR TRUE is False, FALSE IMP FALSE EQV FALSE is True). + | left=expression op=(XOR | BXOR) right=expression # XorExpr + | left=expression op=EQV right=expression # EqvExpr + | left=expression op=IMP right=expression # ImpExpr | primary # PrimaryExpr ; @@ -416,17 +470,22 @@ caseWhen : WHEN condition=expression THEN result=expression ; // a new window function cost NO grammar at all: ROW_NUMBER, RANK, NTILE and friends already lex as IDENTIFIER // and reach here through `functionName`, and `SUM(x) OVER (…)` — an aggregate over a window — parses for free // as the same shape. Access has no window functions; this is a LibRed extension for extended mode. +// WITHIN GROUP gives an ordered-set aggregate (PERCENTILE_CONT, PERCENTILE_DISC, LISTAGG) its ordering, and FILTER +// an aggregate the rows it takes in. FROM FIRST/LAST and RESPECT/IGNORE NULLS sit between the call and OVER, where +// the standard puts them, and only with an OVER: a lone `FROM Last` after a call is the FROM clause naming a table +// called Last, which the lookahead to OVER tells apart. functionCall : name=functionName LPAREN (star=STAR | (distinct=DISTINCT? expression (COMMA expression)*))? RPAREN - (OVER windowSpecification)? + withinGroup? filterClause? + (nthRowFrom? nullTreatment? OVER windowSpecification)? ; // A function name is an identifier, or the LEFT/RIGHT/ASC keywords used as the Left()/Right()/Asc() functions — // unambiguous with LEFT/RIGHT JOIN and ORDER BY ... ASC because a function call is always followed by '(' and // never appears in the FROM/ORDER BY clause. // Keywords that are also function names have to be readmitted here or the lexer's keyword token wins and the // call stops parsing: Left/Right/Asc, and FIRST — which `offsetFetchClause` needs as a keyword for -// `FETCH FIRST`, but which is also the Access aggregate First(). (LAST is not listed because nothing else -// claims it as a keyword.) +// `FETCH FIRST`, but which is also the Access aggregate First(). (LAST is not listed because it is a non-reserved +// keyword, which `identifier` already admits.) // PARTITION is readmitted for the same reason: `PARTITION BY` makes it a keyword, but Access has a real VBA // Partition(number, start, stop, interval) function that LibRed implements and tests. A function call is always // followed by '(' and `PARTITION BY` never is, so the two never collide. @@ -434,7 +493,7 @@ functionName : identifier | LEFT | RIGHT | ASC | FIRST | PARTITION ; columnRef : (qualifier=identifier DOT)? name=identifier ; -identifier : IDENTIFIER | BRACKET_ID | BACKTICK_ID ; +identifier : IDENTIFIER | BRACKET_ID | BACKTICK_ID | nonReservedKeyword ; literal : INTEGER_LITERAL # IntLiteral @@ -465,13 +524,50 @@ standaloneExpression : expression EOF ; // A window function's OVER (…). Both parts are optional here even though EF Core always emits both and the // standard's defaults differ (no PARTITION BY = one partition over the whole input; no ORDER BY = every row a // peer), because rejecting them in the grammar would report a parse error where a semantic one is clearer. -// A frame clause (ROWS/RANGE BETWEEN …) goes before the RPAREN when something needs one — nothing emits one -// today, and admitting it now would reserve five more keywords (RANGE, PRECEDING, FOLLOWING, UNBOUNDED, -// CURRENT) to buy nothing. Kept after the existing parser rules so adding it does not renumber their ids. +// Kept after the existing parser rules so adding it does not renumber their ids. windowSpecification - : LPAREN (PARTITION BY partition+=expression (COMMA partition+=expression)*)? orderByClause? RPAREN + : LPAREN (PARTITION BY partition+=expression (COMMA partition+=expression)*)? orderByClause? windowFrame? RPAREN + ; + +// CLUSTERED / NONCLUSTERED after PRIMARY KEY or UNIQUE — accepted and ignored (see columnConstraint). Kept after +// the existing parser rules so adding it does not renumber their ids. +clusteredOption : CLUSTERED | NONCLUSTERED ; + +// A window frame, as the standard has it: the rows of the partition a frame-reading function (an aggregate, +// FIRST_VALUE, …) sees from the current row. A lone bound is the frame's start, ending at the current row. +// ROWS | RANGE | GROUPS [BETWEEN start AND end | start] [EXCLUDE CURRENT ROW | GROUP | TIES | NO OTHERS] +// UNBOUNDED is listed before the offset form so `UNBOUNDED PRECEDING` is the bound, not a column named Unbounded +// (ANTLR settles an ambiguity on the lower alternative); bracket such a column to use it as an offset. +windowFrame + : unit=(ROWS | RANGE | GROUPS) (BETWEEN start=frameBound AND end=frameBound | start=frameBound) + (EXCLUDE exclusion=frameExclusion)? + ; +frameBound + : UNBOUNDED direction=(PRECEDING | FOLLOWING) + | CURRENT ROW + | offset=expression direction=(PRECEDING | FOLLOWING) + ; +frameExclusion : CURRENT ROW | GROUP | TIES | NO OTHERS ; + +// The window clauses' words are keywords only there: each is also admitted as an identifier, so a column named +// Range or Current keeps working unbracketed, as it does in ACE, which reserves none of them. +nonReservedKeyword + : RANGE | GROUPS | UNBOUNDED | PRECEDING | FOLLOWING | CURRENT | EXCLUDE | TIES | OTHERS + | WITHIN | LAST | RESPECT | NULLS | FILTER ; +// An aggregate's FILTER: only the rows for which the condition is true go into it. +filterClause : FILTER LPAREN WHERE condition=expression RPAREN ; + +// An ordered-set aggregate's ordering: PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY x). +withinGroup : WITHIN GROUP LPAREN orderByClause RPAREN ; + +// NTH_VALUE's direction: counted from the frame's first row (the default) or its last. +nthRowFrom : FROM edge=(FIRST | LAST) ; + +// Whether LAG, LEAD, FIRST_VALUE, LAST_VALUE and NTH_VALUE count the rows whose value is Null (the default) or skip them. +nullTreatment : treatment=(RESPECT | IGNORE) NULLS ; + // ---- Lexer ---- SELECT : [Ss][Ee][Ll][Ee][Cc][Tt] ; @@ -482,6 +578,9 @@ AS : [Aa][Ss] ; AND : [Aa][Nn][Dd] ; OR : [Oo][Rr] ; NOT : [Nn][Oo][Tt] ; +XOR : [Xx][Oo][Rr] ; +EQV : [Ee][Qq][Vv] ; +IMP : [Ii][Mm][Pp] ; BAND : [Bb][Aa][Nn][Dd] ; BOR : [Bb][Oo][Rr] ; BXOR : [Bb][Xx][Oo][Rr] ; @@ -557,6 +656,11 @@ SET : [Ss][Ee][Tt] ; DEFAULT : [Dd][Ee][Ff][Aa][Uu][Ll][Tt] ; NO : [Nn][Oo] ; UNIQUE : [Uu][Nn][Ii][Qq][Uu][Ee] ; +// Reserved as ACE reserves them: neither may name a table, column or alias unbracketed. +CLUSTERED : [Cc][Ll][Uu][Ss][Tt][Ee][Rr][Ee][Dd] ; +// Reserved as ACE reserves it. @@IDENTITY still lexes as one SYSVAR token, the longer match. +IDENTITY : [Ii][Dd][Ee][Nn][Tt][Ii][Tt][Yy] ; +NONCLUSTERED : [Nn][Oo][Nn][Cc][Ll][Uu][Ss][Tt][Ee][Rr][Ee][Dd] ; INDEX : [Ii][Nn][Dd][Ee][Xx] ; TEMPORARY : [Tt][Ee][Mm][Pp][Oo][Rr][Aa][Rr][Yy] ; WITH : [Ww][Ii][Tt][Hh] ; @@ -575,6 +679,21 @@ DESC : [Dd][Ee][Ss][Cc] ; TRUE : [Tt][Rr][Uu][Ee] ; FALSE : [Ff][Aa][Ll][Ss][Ee] ; NULL : [Nn][Uu][Ll][Ll] ; +// The window clauses' words — not reserved; see nonReservedKeyword. +RANGE : [Rr][Aa][Nn][Gg][Ee] ; +GROUPS : [Gg][Rr][Oo][Uu][Pp][Ss] ; +UNBOUNDED : [Uu][Nn][Bb][Oo][Uu][Nn][Dd][Ee][Dd] ; +PRECEDING : [Pp][Rr][Ee][Cc][Ee][Dd][Ii][Nn][Gg] ; +FOLLOWING : [Ff][Oo][Ll][Ll][Oo][Ww][Ii][Nn][Gg] ; +CURRENT : [Cc][Uu][Rr][Rr][Ee][Nn][Tt] ; +EXCLUDE : [Ee][Xx][Cc][Ll][Uu][Dd][Ee] ; +TIES : [Tt][Ii][Ee][Ss] ; +OTHERS : [Oo][Tt][Hh][Ee][Rr][Ss] ; +WITHIN : [Ww][Ii][Tt][Hh][Ii][Nn] ; +LAST : [Ll][Aa][Ss][Tt] ; +RESPECT : [Rr][Ee][Ss][Pp][Ee][Cc][Tt] ; +NULLS : [Nn][Uu][Ll][Ll][Ss] ; +FILTER : [Ff][Ii][Ll][Tt][Ee][Rr] ; STAR : '*' ; SLASH : '/' ; @@ -623,5 +742,7 @@ IDENTIFIER : [A-Za-z_][A-Za-z_0-9]* '$'? ; WS : [ \t\r\n]+ -> skip ; // SQL comments — EF Core query tags prepend a `-- tag` line comment to the statement; also block comments. -LINE_COMMENT : '--' ~[\r\n]* -> skip ; +// '--' starts a comment unless nothing but more dashes stands between it and a number, so '--2' and '---2' are +// repeated negation as in ACE (which has no comments at all), while '--Before' is a comment. +LINE_COMMENT : '--' {!DashesThenNumber()}? ~[\r\n]* -> skip ; BLOCK_COMMENT : '/*' .*? '*/' -> skip ; diff --git a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlBaseVisitor.cs b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlBaseVisitor.cs index ec56f31a3..0401ab57e 100644 --- a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlBaseVisitor.cs +++ b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlBaseVisitor.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -// Generated from AccessSql.g4 by ANTLR 4.13.1 +// Generated from D:/toolkits/efcorejetlibred/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 by ANTLR 4.13.1 // Unreachable code detected #pragma warning disable 0162 @@ -551,6 +551,17 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitColumnReferencesConstraint([NotNull] AccessSqlParser.ColumnReferencesConstraintContext context) { return VisitChildren(context); } /// + /// Visit a parse tree produced by the IdentityConstraint + /// labeled alternative in . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitIdentityConstraint([NotNull] AccessSqlParser.IdentityConstraintContext context) { return VisitChildren(context); } + /// /// Visit a parse tree produced by the PrimaryKeyTableConstraint /// labeled alternative in . /// @@ -1059,6 +1070,17 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitAndExpr([NotNull] AccessSqlParser.AndExprContext context) { return VisitChildren(context); } /// + /// Visit a parse tree produced by the IntDivExpr + /// labeled alternative in . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitIntDivExpr([NotNull] AccessSqlParser.IntDivExprContext context) { return VisitChildren(context); } + /// /// Visit a parse tree produced by the LikeExpr /// labeled alternative in . /// @@ -1092,7 +1114,7 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitComparisonExpr([NotNull] AccessSqlParser.ComparisonExprContext context) { return VisitChildren(context); } /// - /// Visit a parse tree produced by the BetweenExpr + /// Visit a parse tree produced by the XorExpr /// labeled alternative in . /// /// The default implementation returns the result of calling @@ -1101,9 +1123,9 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitor /// The parse tree. /// The visitor result. - public virtual Result VisitBetweenExpr([NotNull] AccessSqlParser.BetweenExprContext context) { return VisitChildren(context); } + public virtual Result VisitXorExpr([NotNull] AccessSqlParser.XorExprContext context) { return VisitChildren(context); } /// - /// Visit a parse tree produced by the BitNotExpr + /// Visit a parse tree produced by the BetweenExpr /// labeled alternative in . /// /// The default implementation returns the result of calling @@ -1112,7 +1134,7 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitor /// The parse tree. /// The visitor result. - public virtual Result VisitBitNotExpr([NotNull] AccessSqlParser.BitNotExprContext context) { return VisitChildren(context); } + public virtual Result VisitBetweenExpr([NotNull] AccessSqlParser.BetweenExprContext context) { return VisitChildren(context); } /// /// Visit a parse tree produced by the PowExpr /// labeled alternative in . @@ -1158,6 +1180,17 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitInSubqueryExpr([NotNull] AccessSqlParser.InSubqueryExprContext context) { return VisitChildren(context); } /// + /// Visit a parse tree produced by the ConcatExpr + /// labeled alternative in . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitConcatExpr([NotNull] AccessSqlParser.ConcatExprContext context) { return VisitChildren(context); } + /// /// Visit a parse tree produced by the MulDivExpr /// labeled alternative in . /// @@ -1169,7 +1202,7 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitMulDivExpr([NotNull] AccessSqlParser.MulDivExprContext context) { return VisitChildren(context); } /// - /// Visit a parse tree produced by the BitwiseExpr + /// Visit a parse tree produced by the ImpExpr /// labeled alternative in . /// /// The default implementation returns the result of calling @@ -1178,7 +1211,7 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitor /// The parse tree. /// The visitor result. - public virtual Result VisitBitwiseExpr([NotNull] AccessSqlParser.BitwiseExprContext context) { return VisitChildren(context); } + public virtual Result VisitImpExpr([NotNull] AccessSqlParser.ImpExprContext context) { return VisitChildren(context); } /// /// Visit a parse tree produced by the PrimaryExpr /// labeled alternative in . @@ -1202,6 +1235,17 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitNotExpr([NotNull] AccessSqlParser.NotExprContext context) { return VisitChildren(context); } /// + /// Visit a parse tree produced by the ModExpr + /// labeled alternative in . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitModExpr([NotNull] AccessSqlParser.ModExprContext context) { return VisitChildren(context); } + /// /// Visit a parse tree produced by the InExpr /// labeled alternative in . /// @@ -1213,7 +1257,18 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe visitor result. public virtual Result VisitInExpr([NotNull] AccessSqlParser.InExprContext context) { return VisitChildren(context); } /// - /// Visit a parse tree produced by the AddConcatExpr + /// Visit a parse tree produced by the AddSubExpr + /// labeled alternative in . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitAddSubExpr([NotNull] AccessSqlParser.AddSubExprContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by the EqvExpr /// labeled alternative in . /// /// The default implementation returns the result of calling @@ -1222,7 +1277,7 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitor /// The parse tree. /// The visitor result. - public virtual Result VisitAddConcatExpr([NotNull] AccessSqlParser.AddConcatExprContext context) { return VisitChildren(context); } + public virtual Result VisitEqvExpr([NotNull] AccessSqlParser.EqvExprContext context) { return VisitChildren(context); } /// /// Visit a parse tree produced by the LiteralPrimary /// labeled alternative in . @@ -1534,5 +1589,95 @@ public partial class AccessSqlBaseVisitor : AbstractParseTreeVisitorThe parse tree. /// The visitor result. public virtual Result VisitWindowSpecification([NotNull] AccessSqlParser.WindowSpecificationContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitClusteredOption([NotNull] AccessSqlParser.ClusteredOptionContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitWindowFrame([NotNull] AccessSqlParser.WindowFrameContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitFrameBound([NotNull] AccessSqlParser.FrameBoundContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitFrameExclusion([NotNull] AccessSqlParser.FrameExclusionContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitNonReservedKeyword([NotNull] AccessSqlParser.NonReservedKeywordContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitFilterClause([NotNull] AccessSqlParser.FilterClauseContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitWithinGroup([NotNull] AccessSqlParser.WithinGroupContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitNthRowFrom([NotNull] AccessSqlParser.NthRowFromContext context) { return VisitChildren(context); } + /// + /// Visit a parse tree produced by . + /// + /// The default implementation returns the result of calling + /// on . + /// + /// + /// The parse tree. + /// The visitor result. + public virtual Result VisitNullTreatment([NotNull] AccessSqlParser.NullTreatmentContext context) { return VisitChildren(context); } } } // namespace LibRed.Sql.Grammar diff --git a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlLexer.cs b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlLexer.cs index 975e1ea36..a4257e173 100644 --- a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlLexer.cs +++ b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlLexer.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -// Generated from AccessSql.g4 by ANTLR 4.13.1 +// Generated from D:/toolkits/efcorejetlibred/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 by ANTLR 4.13.1 // Unreachable code detected #pragma warning disable 0162 @@ -34,25 +34,29 @@ public partial class AccessSqlLexer : Lexer { protected static DFA[] decisionToDFA; protected static PredictionContextCache sharedContextCache = new PredictionContextCache(); public const int - SELECT=1, FROM=2, WHERE=3, TOP=4, AS=5, AND=6, OR=7, NOT=8, BAND=9, BOR=10, - BXOR=11, BNOT=12, LIKE=13, MOD=14, INNER=15, LEFT=16, RIGHT=17, FULL=18, - OUTER=19, JOIN=20, IN=21, ON=22, ORDER=23, GROUP=24, IS=25, BY=26, HAVING=27, - EXISTS=28, IF=29, THEN=30, DISTINCTROW=31, DISTINCT=32, PERCENT=33, CROSS=34, - APPLY=35, OVER=36, PARTITION=37, CASE=38, WHEN=39, ELSE=40, END=41, OFFSET=42, - FETCH=43, NEXT=44, FIRST=45, ROWS=46, ROW=47, ONLY=48, BETWEEN=49, UNION=50, - ALL=51, INTERSECT=52, EXCEPT=53, CREATE=54, TABLE=55, BEGIN=56, COMMIT=57, - ROLLBACK=58, TRANSACTION=59, WORK=60, ALTER=61, RENAME=62, TO=63, ADD=64, - DROP=65, COLUMN=66, INSERT=67, INTO=68, VALUES=69, PRIMARY=70, KEY=71, - CONSTRAINT=72, FOREIGN=73, REFERENCES=74, DELETE=75, UPDATE=76, CASCADE=77, - RESTRICT=78, ACTION=79, SET=80, DEFAULT=81, NO=82, UNIQUE=83, INDEX=84, - TEMPORARY=85, WITH=86, COMPRESSION=87, COMP=88, DISALLOW=89, IGNORE=90, - CHECK=91, VIEW=92, PROCEDURE=93, PARAMETERS=94, EXECUTE=95, EXEC=96, ASC=97, - DESC=98, TRUE=99, FALSE=100, NULL=101, STAR=102, SLASH=103, BACKSLASH=104, - CARET=105, PLUS=106, MINUS=107, AMP=108, EQ=109, NEQ=110, LTE=111, GTE=112, - LT=113, GT=114, LPAREN=115, RPAREN=116, COMMA=117, DOT=118, SEMI=119, - SYSVAR=120, PARAM=121, HEX_LITERAL=122, INTEGER_LITERAL=123, NUMBER_LITERAL=124, - STRING_LITERAL=125, DATE_LITERAL=126, GUID_LITERAL=127, BRACKET_ID=128, - BACKTICK_ID=129, IDENTIFIER=130, WS=131, LINE_COMMENT=132, BLOCK_COMMENT=133; + SELECT=1, FROM=2, WHERE=3, TOP=4, AS=5, AND=6, OR=7, NOT=8, XOR=9, EQV=10, + IMP=11, BAND=12, BOR=13, BXOR=14, BNOT=15, LIKE=16, MOD=17, INNER=18, + LEFT=19, RIGHT=20, FULL=21, OUTER=22, JOIN=23, IN=24, ON=25, ORDER=26, + GROUP=27, IS=28, BY=29, HAVING=30, EXISTS=31, IF=32, THEN=33, DISTINCTROW=34, + DISTINCT=35, PERCENT=36, CROSS=37, APPLY=38, OVER=39, PARTITION=40, CASE=41, + WHEN=42, ELSE=43, END=44, OFFSET=45, FETCH=46, NEXT=47, FIRST=48, ROWS=49, + ROW=50, ONLY=51, BETWEEN=52, UNION=53, ALL=54, INTERSECT=55, EXCEPT=56, + CREATE=57, TABLE=58, BEGIN=59, COMMIT=60, ROLLBACK=61, TRANSACTION=62, + WORK=63, ALTER=64, RENAME=65, TO=66, ADD=67, DROP=68, COLUMN=69, INSERT=70, + INTO=71, VALUES=72, PRIMARY=73, KEY=74, CONSTRAINT=75, FOREIGN=76, REFERENCES=77, + DELETE=78, UPDATE=79, CASCADE=80, RESTRICT=81, ACTION=82, SET=83, DEFAULT=84, + NO=85, UNIQUE=86, CLUSTERED=87, IDENTITY=88, NONCLUSTERED=89, INDEX=90, + TEMPORARY=91, WITH=92, COMPRESSION=93, COMP=94, DISALLOW=95, IGNORE=96, + CHECK=97, VIEW=98, PROCEDURE=99, PARAMETERS=100, EXECUTE=101, EXEC=102, + ASC=103, DESC=104, TRUE=105, FALSE=106, NULL=107, RANGE=108, GROUPS=109, + UNBOUNDED=110, PRECEDING=111, FOLLOWING=112, CURRENT=113, EXCLUDE=114, + TIES=115, OTHERS=116, WITHIN=117, LAST=118, RESPECT=119, NULLS=120, FILTER=121, + STAR=122, SLASH=123, BACKSLASH=124, CARET=125, PLUS=126, MINUS=127, AMP=128, + EQ=129, NEQ=130, LTE=131, GTE=132, LT=133, GT=134, LPAREN=135, RPAREN=136, + COMMA=137, DOT=138, SEMI=139, SYSVAR=140, PARAM=141, HEX_LITERAL=142, + INTEGER_LITERAL=143, NUMBER_LITERAL=144, STRING_LITERAL=145, DATE_LITERAL=146, + GUID_LITERAL=147, BRACKET_ID=148, BACKTICK_ID=149, IDENTIFIER=150, WS=151, + LINE_COMMENT=152, BLOCK_COMMENT=153; public static string[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; @@ -62,27 +66,41 @@ public const int }; public static readonly string[] ruleNames = { - "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "BAND", "BOR", - "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", "RIGHT", "FULL", "OUTER", - "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", "BY", "HAVING", "EXISTS", - "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", "CROSS", "APPLY", - "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", "OFFSET", "FETCH", - "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", "UNION", "ALL", "INTERSECT", - "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", "ROLLBACK", "TRANSACTION", - "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", "COLUMN", "INSERT", "INTO", - "VALUES", "PRIMARY", "KEY", "CONSTRAINT", "FOREIGN", "REFERENCES", "DELETE", - "UPDATE", "CASCADE", "RESTRICT", "ACTION", "SET", "DEFAULT", "NO", "UNIQUE", + "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "XOR", "EQV", + "IMP", "BAND", "BOR", "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", + "RIGHT", "FULL", "OUTER", "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", + "BY", "HAVING", "EXISTS", "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", + "CROSS", "APPLY", "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", + "OFFSET", "FETCH", "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", + "UNION", "ALL", "INTERSECT", "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", + "ROLLBACK", "TRANSACTION", "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", + "COLUMN", "INSERT", "INTO", "VALUES", "PRIMARY", "KEY", "CONSTRAINT", + "FOREIGN", "REFERENCES", "DELETE", "UPDATE", "CASCADE", "RESTRICT", "ACTION", + "SET", "DEFAULT", "NO", "UNIQUE", "CLUSTERED", "IDENTITY", "NONCLUSTERED", "INDEX", "TEMPORARY", "WITH", "COMPRESSION", "COMP", "DISALLOW", "IGNORE", "CHECK", "VIEW", "PROCEDURE", "PARAMETERS", "EXECUTE", "EXEC", "ASC", - "DESC", "TRUE", "FALSE", "NULL", "STAR", "SLASH", "BACKSLASH", "CARET", - "PLUS", "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", - "RPAREN", "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", + "DESC", "TRUE", "FALSE", "NULL", "RANGE", "GROUPS", "UNBOUNDED", "PRECEDING", + "FOLLOWING", "CURRENT", "EXCLUDE", "TIES", "OTHERS", "WITHIN", "LAST", + "RESPECT", "NULLS", "FILTER", "STAR", "SLASH", "BACKSLASH", "CARET", "PLUS", + "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", "RPAREN", + "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", "NUMBER_LITERAL", "EXPONENT", "STRING_LITERAL", "DATE_LITERAL", "GUID_LITERAL", "HEXDIGIT", "BRACKET_ID", "BACKTICK_ID", "IDENTIFIER", "WS", "LINE_COMMENT", "BLOCK_COMMENT" }; + // Whether the input ahead is any number of dashes and then a number: a digit, or a point and a digit. + private bool DashesThenNumber() + { + int i = 1; + while (InputStream.LA(i) == '-') + i++; + int c = InputStream.LA(i); + return c is >= '0' and <= '9' || c == '.' && InputStream.LA(i + 1) is >= '0' and <= '9'; + } + + public AccessSqlLexer(ICharStream input) : this(input, Console.Out, Console.Error) { } @@ -101,26 +119,30 @@ public AccessSqlLexer(ICharStream input, TextWriter output, TextWriter errorOutp null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, "'*'", "'/'", "'\\'", "'^'", "'+'", - "'-'", "'&'", "'='", null, "'<='", "'>='", "'<'", "'>'", "'('", "')'", - "','", "'.'", "';'" + null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, + null, null, "'*'", "'/'", "'\\'", "'^'", "'+'", "'-'", "'&'", "'='", null, + "'<='", "'>='", "'<'", "'>'", "'('", "')'", "','", "'.'", "';'" }; private static readonly string[] _SymbolicNames = { - null, "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "BAND", - "BOR", "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", "RIGHT", "FULL", - "OUTER", "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", "BY", "HAVING", "EXISTS", - "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", "CROSS", "APPLY", - "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", "OFFSET", "FETCH", - "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", "UNION", "ALL", "INTERSECT", - "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", "ROLLBACK", "TRANSACTION", - "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", "COLUMN", "INSERT", "INTO", - "VALUES", "PRIMARY", "KEY", "CONSTRAINT", "FOREIGN", "REFERENCES", "DELETE", - "UPDATE", "CASCADE", "RESTRICT", "ACTION", "SET", "DEFAULT", "NO", "UNIQUE", + null, "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "XOR", + "EQV", "IMP", "BAND", "BOR", "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", + "RIGHT", "FULL", "OUTER", "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", + "BY", "HAVING", "EXISTS", "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", + "CROSS", "APPLY", "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", + "OFFSET", "FETCH", "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", + "UNION", "ALL", "INTERSECT", "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", + "ROLLBACK", "TRANSACTION", "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", + "COLUMN", "INSERT", "INTO", "VALUES", "PRIMARY", "KEY", "CONSTRAINT", + "FOREIGN", "REFERENCES", "DELETE", "UPDATE", "CASCADE", "RESTRICT", "ACTION", + "SET", "DEFAULT", "NO", "UNIQUE", "CLUSTERED", "IDENTITY", "NONCLUSTERED", "INDEX", "TEMPORARY", "WITH", "COMPRESSION", "COMP", "DISALLOW", "IGNORE", "CHECK", "VIEW", "PROCEDURE", "PARAMETERS", "EXECUTE", "EXEC", "ASC", - "DESC", "TRUE", "FALSE", "NULL", "STAR", "SLASH", "BACKSLASH", "CARET", - "PLUS", "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", - "RPAREN", "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", + "DESC", "TRUE", "FALSE", "NULL", "RANGE", "GROUPS", "UNBOUNDED", "PRECEDING", + "FOLLOWING", "CURRENT", "EXCLUDE", "TIES", "OTHERS", "WITHIN", "LAST", + "RESPECT", "NULLS", "FILTER", "STAR", "SLASH", "BACKSLASH", "CARET", "PLUS", + "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", "RPAREN", + "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", "NUMBER_LITERAL", "STRING_LITERAL", "DATE_LITERAL", "GUID_LITERAL", "BRACKET_ID", "BACKTICK_ID", "IDENTIFIER", "WS", "LINE_COMMENT", "BLOCK_COMMENT" }; @@ -151,8 +173,21 @@ static AccessSqlLexer() { decisionToDFA[i] = new DFA(_ATN.GetDecisionState(i), i); } } + public override bool Sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + switch (ruleIndex) { + case 153 : return LINE_COMMENT_sempred(_localctx, predIndex); + } + return true; + } + private bool LINE_COMMENT_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 0: return !DashesThenNumber(); + } + return true; + } + private static int[] _serializedATN = { - 4,0,133,1130,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6, + 4,0,153,1319,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6, 7,6,2,7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,2,11,7,11,2,12,7,12,2,13,7,13,2, 14,7,14,2,15,7,15,2,16,7,16,2,17,7,17,2,18,7,18,2,19,7,19,2,20,7,20,2, 21,7,21,2,22,7,22,2,23,7,23,2,24,7,24,2,25,7,25,2,26,7,26,2,27,7,27,2, @@ -172,364 +207,429 @@ static AccessSqlLexer() { 7,116,2,117,7,117,2,118,7,118,2,119,7,119,2,120,7,120,2,121,7,121,2,122, 7,122,2,123,7,123,2,124,7,124,2,125,7,125,2,126,7,126,2,127,7,127,2,128, 7,128,2,129,7,129,2,130,7,130,2,131,7,131,2,132,7,132,2,133,7,133,2,134, - 7,134,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,2,1,2,1,2,1,2, - 1,2,1,2,1,3,1,3,1,3,1,3,1,4,1,4,1,4,1,5,1,5,1,5,1,5,1,6,1,6,1,6,1,7,1, - 7,1,7,1,7,1,8,1,8,1,8,1,8,1,8,1,9,1,9,1,9,1,9,1,10,1,10,1,10,1,10,1,10, - 1,11,1,11,1,11,1,11,1,11,1,12,1,12,1,12,1,12,1,12,1,13,1,13,1,13,1,13, - 1,14,1,14,1,14,1,14,1,14,1,14,1,15,1,15,1,15,1,15,1,15,1,16,1,16,1,16, - 1,16,1,16,1,16,1,17,1,17,1,17,1,17,1,17,1,18,1,18,1,18,1,18,1,18,1,18, - 1,19,1,19,1,19,1,19,1,19,1,20,1,20,1,20,1,21,1,21,1,21,1,22,1,22,1,22, - 1,22,1,22,1,22,1,23,1,23,1,23,1,23,1,23,1,23,1,24,1,24,1,24,1,25,1,25, - 1,25,1,26,1,26,1,26,1,26,1,26,1,26,1,26,1,27,1,27,1,27,1,27,1,27,1,27, - 1,27,1,28,1,28,1,28,1,29,1,29,1,29,1,29,1,29,1,30,1,30,1,30,1,30,1,30, - 1,30,1,30,1,30,1,30,1,30,1,30,1,30,1,31,1,31,1,31,1,31,1,31,1,31,1,31, - 1,31,1,31,1,32,1,32,1,32,1,32,1,32,1,32,1,32,1,32,1,33,1,33,1,33,1,33, - 1,33,1,33,1,34,1,34,1,34,1,34,1,34,1,34,1,35,1,35,1,35,1,35,1,35,1,36, - 1,36,1,36,1,36,1,36,1,36,1,36,1,36,1,36,1,36,1,37,1,37,1,37,1,37,1,37, - 1,38,1,38,1,38,1,38,1,38,1,39,1,39,1,39,1,39,1,39,1,40,1,40,1,40,1,40, - 1,41,1,41,1,41,1,41,1,41,1,41,1,41,1,42,1,42,1,42,1,42,1,42,1,42,1,43, - 1,43,1,43,1,43,1,43,1,44,1,44,1,44,1,44,1,44,1,44,1,45,1,45,1,45,1,45, - 1,45,1,46,1,46,1,46,1,46,1,47,1,47,1,47,1,47,1,47,1,48,1,48,1,48,1,48, - 1,48,1,48,1,48,1,48,1,49,1,49,1,49,1,49,1,49,1,49,1,50,1,50,1,50,1,50, - 1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,52,1,52,1,52,1,52, - 1,52,1,52,1,52,1,53,1,53,1,53,1,53,1,53,1,53,1,53,1,54,1,54,1,54,1,54, - 1,54,1,54,1,55,1,55,1,55,1,55,1,55,1,55,1,56,1,56,1,56,1,56,1,56,1,56, - 1,56,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,58,1,58,1,58,1,58, - 1,58,1,58,1,58,1,58,1,58,1,58,1,58,1,58,1,59,1,59,1,59,1,59,1,59,1,60, - 1,60,1,60,1,60,1,60,1,60,1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,62,1,62, - 1,62,1,63,1,63,1,63,1,63,1,64,1,64,1,64,1,64,1,64,1,65,1,65,1,65,1,65, - 1,65,1,65,1,65,1,66,1,66,1,66,1,66,1,66,1,66,1,66,1,67,1,67,1,67,1,67, - 1,67,1,68,1,68,1,68,1,68,1,68,1,68,1,68,1,69,1,69,1,69,1,69,1,69,1,69, - 1,69,1,69,1,70,1,70,1,70,1,70,1,71,1,71,1,71,1,71,1,71,1,71,1,71,1,71, - 1,71,1,71,1,71,1,72,1,72,1,72,1,72,1,72,1,72,1,72,1,72,1,73,1,73,1,73, - 1,73,1,73,1,73,1,73,1,73,1,73,1,73,1,73,1,74,1,74,1,74,1,74,1,74,1,74, - 1,74,1,75,1,75,1,75,1,75,1,75,1,75,1,75,1,76,1,76,1,76,1,76,1,76,1,76, - 1,76,1,76,1,77,1,77,1,77,1,77,1,77,1,77,1,77,1,77,1,77,1,78,1,78,1,78, - 1,78,1,78,1,78,1,78,1,79,1,79,1,79,1,79,1,80,1,80,1,80,1,80,1,80,1,80, - 1,80,1,80,1,81,1,81,1,81,1,82,1,82,1,82,1,82,1,82,1,82,1,82,1,83,1,83, - 1,83,1,83,1,83,1,83,1,84,1,84,1,84,1,84,1,84,1,84,1,84,1,84,1,84,1,84, - 1,85,1,85,1,85,1,85,1,85,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,86, - 1,86,1,86,1,86,1,87,1,87,1,87,1,87,1,87,1,88,1,88,1,88,1,88,1,88,1,88, - 1,88,1,88,1,88,1,89,1,89,1,89,1,89,1,89,1,89,1,89,1,90,1,90,1,90,1,90, - 1,90,1,90,1,91,1,91,1,91,1,91,1,91,1,92,1,92,1,92,1,92,1,92,1,92,1,92, - 1,92,1,92,1,92,1,93,1,93,1,93,1,93,1,93,1,93,1,93,1,93,1,93,1,93,1,93, - 1,94,1,94,1,94,1,94,1,94,1,94,1,94,1,94,1,95,1,95,1,95,1,95,1,95,1,96, - 1,96,1,96,1,96,1,97,1,97,1,97,1,97,1,97,1,98,1,98,1,98,1,98,1,98,1,99, - 1,99,1,99,1,99,1,99,1,99,1,100,1,100,1,100,1,100,1,100,1,101,1,101,1,102, - 1,102,1,103,1,103,1,104,1,104,1,105,1,105,1,106,1,106,1,107,1,107,1,108, - 1,108,1,109,1,109,1,109,1,109,3,109,912,8,109,1,110,1,110,1,110,1,111, - 1,111,1,111,1,112,1,112,1,113,1,113,1,114,1,114,1,115,1,115,1,116,1,116, - 1,117,1,117,1,118,1,118,1,119,1,119,1,119,1,119,1,119,5,119,939,8,119, - 10,119,12,119,942,9,119,1,120,1,120,1,120,1,120,5,120,948,8,120,10,120, - 12,120,951,9,120,3,120,953,8,120,1,121,1,121,1,121,4,121,958,8,121,11, - 121,12,121,959,1,122,4,122,963,8,122,11,122,12,122,964,1,123,4,123,968, - 8,123,11,123,12,123,969,1,123,1,123,5,123,974,8,123,10,123,12,123,977, - 9,123,1,123,3,123,980,8,123,1,123,1,123,4,123,984,8,123,11,123,12,123, - 985,1,123,3,123,989,8,123,1,123,4,123,992,8,123,11,123,12,123,993,1,123, - 3,123,997,8,123,1,124,1,124,3,124,1001,8,124,1,124,4,124,1004,8,124,11, - 124,12,124,1005,1,125,1,125,1,125,1,125,5,125,1012,8,125,10,125,12,125, - 1015,9,125,1,125,1,125,1,125,1,125,1,125,5,125,1022,8,125,10,125,12,125, - 1025,9,125,1,125,3,125,1028,8,125,1,126,1,126,5,126,1032,8,126,10,126, - 12,126,1035,9,126,1,126,1,126,1,127,1,127,4,127,1041,8,127,11,127,12,127, - 1042,1,127,1,127,4,127,1047,8,127,11,127,12,127,1048,1,127,1,127,4,127, - 1053,8,127,11,127,12,127,1054,1,127,1,127,4,127,1059,8,127,11,127,12,127, - 1060,1,127,1,127,4,127,1065,8,127,11,127,12,127,1066,1,127,1,127,1,128, - 1,128,1,129,1,129,4,129,1075,8,129,11,129,12,129,1076,1,129,1,129,1,130, - 1,130,4,130,1083,8,130,11,130,12,130,1084,1,130,1,130,1,131,1,131,5,131, - 1091,8,131,10,131,12,131,1094,9,131,1,131,3,131,1097,8,131,1,132,4,132, - 1100,8,132,11,132,12,132,1101,1,132,1,132,1,133,1,133,1,133,1,133,5,133, - 1110,8,133,10,133,12,133,1113,9,133,1,133,1,133,1,134,1,134,1,134,1,134, - 5,134,1121,8,134,10,134,12,134,1124,9,134,1,134,1,134,1,134,1,134,1,134, - 1,1122,0,135,1,1,3,2,5,3,7,4,9,5,11,6,13,7,15,8,17,9,19,10,21,11,23,12, - 25,13,27,14,29,15,31,16,33,17,35,18,37,19,39,20,41,21,43,22,45,23,47,24, - 49,25,51,26,53,27,55,28,57,29,59,30,61,31,63,32,65,33,67,34,69,35,71,36, - 73,37,75,38,77,39,79,40,81,41,83,42,85,43,87,44,89,45,91,46,93,47,95,48, - 97,49,99,50,101,51,103,52,105,53,107,54,109,55,111,56,113,57,115,58,117, - 59,119,60,121,61,123,62,125,63,127,64,129,65,131,66,133,67,135,68,137, - 69,139,70,141,71,143,72,145,73,147,74,149,75,151,76,153,77,155,78,157, - 79,159,80,161,81,163,82,165,83,167,84,169,85,171,86,173,87,175,88,177, - 89,179,90,181,91,183,92,185,93,187,94,189,95,191,96,193,97,195,98,197, - 99,199,100,201,101,203,102,205,103,207,104,209,105,211,106,213,107,215, - 108,217,109,219,110,221,111,223,112,225,113,227,114,229,115,231,116,233, - 117,235,118,237,119,239,120,241,121,243,122,245,123,247,124,249,0,251, - 125,253,126,255,127,257,0,259,128,261,129,263,130,265,131,267,132,269, - 133,1,0,37,2,0,83,83,115,115,2,0,69,69,101,101,2,0,76,76,108,108,2,0,67, - 67,99,99,2,0,84,84,116,116,2,0,70,70,102,102,2,0,82,82,114,114,2,0,79, - 79,111,111,2,0,77,77,109,109,2,0,87,87,119,119,2,0,72,72,104,104,2,0,80, - 80,112,112,2,0,65,65,97,97,2,0,78,78,110,110,2,0,68,68,100,100,2,0,66, - 66,98,98,2,0,88,88,120,120,2,0,73,73,105,105,2,0,75,75,107,107,2,0,71, - 71,103,103,2,0,85,85,117,117,2,0,74,74,106,106,2,0,89,89,121,121,2,0,86, - 86,118,118,2,0,81,81,113,113,3,0,65,90,95,95,97,122,4,0,48,57,65,90,95, - 95,97,122,3,0,48,57,65,70,97,102,1,0,48,57,2,0,43,43,45,45,1,0,34,34,1, - 0,39,39,1,0,35,35,1,0,93,93,1,0,96,96,3,0,9,10,13,13,32,32,2,0,10,10,13, - 13,1161,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,9,1,0,0,0,0, - 11,1,0,0,0,0,13,1,0,0,0,0,15,1,0,0,0,0,17,1,0,0,0,0,19,1,0,0,0,0,21,1, - 0,0,0,0,23,1,0,0,0,0,25,1,0,0,0,0,27,1,0,0,0,0,29,1,0,0,0,0,31,1,0,0,0, - 0,33,1,0,0,0,0,35,1,0,0,0,0,37,1,0,0,0,0,39,1,0,0,0,0,41,1,0,0,0,0,43, - 1,0,0,0,0,45,1,0,0,0,0,47,1,0,0,0,0,49,1,0,0,0,0,51,1,0,0,0,0,53,1,0,0, - 0,0,55,1,0,0,0,0,57,1,0,0,0,0,59,1,0,0,0,0,61,1,0,0,0,0,63,1,0,0,0,0,65, - 1,0,0,0,0,67,1,0,0,0,0,69,1,0,0,0,0,71,1,0,0,0,0,73,1,0,0,0,0,75,1,0,0, - 0,0,77,1,0,0,0,0,79,1,0,0,0,0,81,1,0,0,0,0,83,1,0,0,0,0,85,1,0,0,0,0,87, - 1,0,0,0,0,89,1,0,0,0,0,91,1,0,0,0,0,93,1,0,0,0,0,95,1,0,0,0,0,97,1,0,0, - 0,0,99,1,0,0,0,0,101,1,0,0,0,0,103,1,0,0,0,0,105,1,0,0,0,0,107,1,0,0,0, - 0,109,1,0,0,0,0,111,1,0,0,0,0,113,1,0,0,0,0,115,1,0,0,0,0,117,1,0,0,0, - 0,119,1,0,0,0,0,121,1,0,0,0,0,123,1,0,0,0,0,125,1,0,0,0,0,127,1,0,0,0, - 0,129,1,0,0,0,0,131,1,0,0,0,0,133,1,0,0,0,0,135,1,0,0,0,0,137,1,0,0,0, - 0,139,1,0,0,0,0,141,1,0,0,0,0,143,1,0,0,0,0,145,1,0,0,0,0,147,1,0,0,0, - 0,149,1,0,0,0,0,151,1,0,0,0,0,153,1,0,0,0,0,155,1,0,0,0,0,157,1,0,0,0, - 0,159,1,0,0,0,0,161,1,0,0,0,0,163,1,0,0,0,0,165,1,0,0,0,0,167,1,0,0,0, - 0,169,1,0,0,0,0,171,1,0,0,0,0,173,1,0,0,0,0,175,1,0,0,0,0,177,1,0,0,0, - 0,179,1,0,0,0,0,181,1,0,0,0,0,183,1,0,0,0,0,185,1,0,0,0,0,187,1,0,0,0, - 0,189,1,0,0,0,0,191,1,0,0,0,0,193,1,0,0,0,0,195,1,0,0,0,0,197,1,0,0,0, - 0,199,1,0,0,0,0,201,1,0,0,0,0,203,1,0,0,0,0,205,1,0,0,0,0,207,1,0,0,0, - 0,209,1,0,0,0,0,211,1,0,0,0,0,213,1,0,0,0,0,215,1,0,0,0,0,217,1,0,0,0, - 0,219,1,0,0,0,0,221,1,0,0,0,0,223,1,0,0,0,0,225,1,0,0,0,0,227,1,0,0,0, - 0,229,1,0,0,0,0,231,1,0,0,0,0,233,1,0,0,0,0,235,1,0,0,0,0,237,1,0,0,0, - 0,239,1,0,0,0,0,241,1,0,0,0,0,243,1,0,0,0,0,245,1,0,0,0,0,247,1,0,0,0, - 0,251,1,0,0,0,0,253,1,0,0,0,0,255,1,0,0,0,0,259,1,0,0,0,0,261,1,0,0,0, - 0,263,1,0,0,0,0,265,1,0,0,0,0,267,1,0,0,0,0,269,1,0,0,0,1,271,1,0,0,0, - 3,278,1,0,0,0,5,283,1,0,0,0,7,289,1,0,0,0,9,293,1,0,0,0,11,296,1,0,0,0, - 13,300,1,0,0,0,15,303,1,0,0,0,17,307,1,0,0,0,19,312,1,0,0,0,21,316,1,0, - 0,0,23,321,1,0,0,0,25,326,1,0,0,0,27,331,1,0,0,0,29,335,1,0,0,0,31,341, - 1,0,0,0,33,346,1,0,0,0,35,352,1,0,0,0,37,357,1,0,0,0,39,363,1,0,0,0,41, - 368,1,0,0,0,43,371,1,0,0,0,45,374,1,0,0,0,47,380,1,0,0,0,49,386,1,0,0, - 0,51,389,1,0,0,0,53,392,1,0,0,0,55,399,1,0,0,0,57,406,1,0,0,0,59,409,1, - 0,0,0,61,414,1,0,0,0,63,426,1,0,0,0,65,435,1,0,0,0,67,443,1,0,0,0,69,449, - 1,0,0,0,71,455,1,0,0,0,73,460,1,0,0,0,75,470,1,0,0,0,77,475,1,0,0,0,79, - 480,1,0,0,0,81,485,1,0,0,0,83,489,1,0,0,0,85,496,1,0,0,0,87,502,1,0,0, - 0,89,507,1,0,0,0,91,513,1,0,0,0,93,518,1,0,0,0,95,522,1,0,0,0,97,527,1, - 0,0,0,99,535,1,0,0,0,101,541,1,0,0,0,103,545,1,0,0,0,105,555,1,0,0,0,107, - 562,1,0,0,0,109,569,1,0,0,0,111,575,1,0,0,0,113,581,1,0,0,0,115,588,1, - 0,0,0,117,597,1,0,0,0,119,609,1,0,0,0,121,614,1,0,0,0,123,620,1,0,0,0, - 125,627,1,0,0,0,127,630,1,0,0,0,129,634,1,0,0,0,131,639,1,0,0,0,133,646, - 1,0,0,0,135,653,1,0,0,0,137,658,1,0,0,0,139,665,1,0,0,0,141,673,1,0,0, - 0,143,677,1,0,0,0,145,688,1,0,0,0,147,696,1,0,0,0,149,707,1,0,0,0,151, - 714,1,0,0,0,153,721,1,0,0,0,155,729,1,0,0,0,157,738,1,0,0,0,159,745,1, - 0,0,0,161,749,1,0,0,0,163,757,1,0,0,0,165,760,1,0,0,0,167,767,1,0,0,0, - 169,773,1,0,0,0,171,783,1,0,0,0,173,788,1,0,0,0,175,800,1,0,0,0,177,805, - 1,0,0,0,179,814,1,0,0,0,181,821,1,0,0,0,183,827,1,0,0,0,185,832,1,0,0, - 0,187,842,1,0,0,0,189,853,1,0,0,0,191,861,1,0,0,0,193,866,1,0,0,0,195, - 870,1,0,0,0,197,875,1,0,0,0,199,880,1,0,0,0,201,886,1,0,0,0,203,891,1, - 0,0,0,205,893,1,0,0,0,207,895,1,0,0,0,209,897,1,0,0,0,211,899,1,0,0,0, - 213,901,1,0,0,0,215,903,1,0,0,0,217,905,1,0,0,0,219,911,1,0,0,0,221,913, - 1,0,0,0,223,916,1,0,0,0,225,919,1,0,0,0,227,921,1,0,0,0,229,923,1,0,0, - 0,231,925,1,0,0,0,233,927,1,0,0,0,235,929,1,0,0,0,237,931,1,0,0,0,239, - 933,1,0,0,0,241,952,1,0,0,0,243,954,1,0,0,0,245,962,1,0,0,0,247,996,1, - 0,0,0,249,998,1,0,0,0,251,1027,1,0,0,0,253,1029,1,0,0,0,255,1038,1,0,0, - 0,257,1070,1,0,0,0,259,1072,1,0,0,0,261,1080,1,0,0,0,263,1088,1,0,0,0, - 265,1099,1,0,0,0,267,1105,1,0,0,0,269,1116,1,0,0,0,271,272,7,0,0,0,272, - 273,7,1,0,0,273,274,7,2,0,0,274,275,7,1,0,0,275,276,7,3,0,0,276,277,7, - 4,0,0,277,2,1,0,0,0,278,279,7,5,0,0,279,280,7,6,0,0,280,281,7,7,0,0,281, - 282,7,8,0,0,282,4,1,0,0,0,283,284,7,9,0,0,284,285,7,10,0,0,285,286,7,1, - 0,0,286,287,7,6,0,0,287,288,7,1,0,0,288,6,1,0,0,0,289,290,7,4,0,0,290, - 291,7,7,0,0,291,292,7,11,0,0,292,8,1,0,0,0,293,294,7,12,0,0,294,295,7, - 0,0,0,295,10,1,0,0,0,296,297,7,12,0,0,297,298,7,13,0,0,298,299,7,14,0, - 0,299,12,1,0,0,0,300,301,7,7,0,0,301,302,7,6,0,0,302,14,1,0,0,0,303,304, - 7,13,0,0,304,305,7,7,0,0,305,306,7,4,0,0,306,16,1,0,0,0,307,308,7,15,0, - 0,308,309,7,12,0,0,309,310,7,13,0,0,310,311,7,14,0,0,311,18,1,0,0,0,312, - 313,7,15,0,0,313,314,7,7,0,0,314,315,7,6,0,0,315,20,1,0,0,0,316,317,7, - 15,0,0,317,318,7,16,0,0,318,319,7,7,0,0,319,320,7,6,0,0,320,22,1,0,0,0, - 321,322,7,15,0,0,322,323,7,13,0,0,323,324,7,7,0,0,324,325,7,4,0,0,325, - 24,1,0,0,0,326,327,7,2,0,0,327,328,7,17,0,0,328,329,7,18,0,0,329,330,7, - 1,0,0,330,26,1,0,0,0,331,332,7,8,0,0,332,333,7,7,0,0,333,334,7,14,0,0, - 334,28,1,0,0,0,335,336,7,17,0,0,336,337,7,13,0,0,337,338,7,13,0,0,338, - 339,7,1,0,0,339,340,7,6,0,0,340,30,1,0,0,0,341,342,7,2,0,0,342,343,7,1, - 0,0,343,344,7,5,0,0,344,345,7,4,0,0,345,32,1,0,0,0,346,347,7,6,0,0,347, - 348,7,17,0,0,348,349,7,19,0,0,349,350,7,10,0,0,350,351,7,4,0,0,351,34, - 1,0,0,0,352,353,7,5,0,0,353,354,7,20,0,0,354,355,7,2,0,0,355,356,7,2,0, - 0,356,36,1,0,0,0,357,358,7,7,0,0,358,359,7,20,0,0,359,360,7,4,0,0,360, - 361,7,1,0,0,361,362,7,6,0,0,362,38,1,0,0,0,363,364,7,21,0,0,364,365,7, - 7,0,0,365,366,7,17,0,0,366,367,7,13,0,0,367,40,1,0,0,0,368,369,7,17,0, - 0,369,370,7,13,0,0,370,42,1,0,0,0,371,372,7,7,0,0,372,373,7,13,0,0,373, - 44,1,0,0,0,374,375,7,7,0,0,375,376,7,6,0,0,376,377,7,14,0,0,377,378,7, - 1,0,0,378,379,7,6,0,0,379,46,1,0,0,0,380,381,7,19,0,0,381,382,7,6,0,0, - 382,383,7,7,0,0,383,384,7,20,0,0,384,385,7,11,0,0,385,48,1,0,0,0,386,387, - 7,17,0,0,387,388,7,0,0,0,388,50,1,0,0,0,389,390,7,15,0,0,390,391,7,22, - 0,0,391,52,1,0,0,0,392,393,7,10,0,0,393,394,7,12,0,0,394,395,7,23,0,0, - 395,396,7,17,0,0,396,397,7,13,0,0,397,398,7,19,0,0,398,54,1,0,0,0,399, - 400,7,1,0,0,400,401,7,16,0,0,401,402,7,17,0,0,402,403,7,0,0,0,403,404, - 7,4,0,0,404,405,7,0,0,0,405,56,1,0,0,0,406,407,7,17,0,0,407,408,7,5,0, - 0,408,58,1,0,0,0,409,410,7,4,0,0,410,411,7,10,0,0,411,412,7,1,0,0,412, - 413,7,13,0,0,413,60,1,0,0,0,414,415,7,14,0,0,415,416,7,17,0,0,416,417, - 7,0,0,0,417,418,7,4,0,0,418,419,7,17,0,0,419,420,7,13,0,0,420,421,7,3, - 0,0,421,422,7,4,0,0,422,423,7,6,0,0,423,424,7,7,0,0,424,425,7,9,0,0,425, - 62,1,0,0,0,426,427,7,14,0,0,427,428,7,17,0,0,428,429,7,0,0,0,429,430,7, - 4,0,0,430,431,7,17,0,0,431,432,7,13,0,0,432,433,7,3,0,0,433,434,7,4,0, - 0,434,64,1,0,0,0,435,436,7,11,0,0,436,437,7,1,0,0,437,438,7,6,0,0,438, - 439,7,3,0,0,439,440,7,1,0,0,440,441,7,13,0,0,441,442,7,4,0,0,442,66,1, - 0,0,0,443,444,7,3,0,0,444,445,7,6,0,0,445,446,7,7,0,0,446,447,7,0,0,0, - 447,448,7,0,0,0,448,68,1,0,0,0,449,450,7,12,0,0,450,451,7,11,0,0,451,452, - 7,11,0,0,452,453,7,2,0,0,453,454,7,22,0,0,454,70,1,0,0,0,455,456,7,7,0, - 0,456,457,7,23,0,0,457,458,7,1,0,0,458,459,7,6,0,0,459,72,1,0,0,0,460, - 461,7,11,0,0,461,462,7,12,0,0,462,463,7,6,0,0,463,464,7,4,0,0,464,465, - 7,17,0,0,465,466,7,4,0,0,466,467,7,17,0,0,467,468,7,7,0,0,468,469,7,13, - 0,0,469,74,1,0,0,0,470,471,7,3,0,0,471,472,7,12,0,0,472,473,7,0,0,0,473, - 474,7,1,0,0,474,76,1,0,0,0,475,476,7,9,0,0,476,477,7,10,0,0,477,478,7, - 1,0,0,478,479,7,13,0,0,479,78,1,0,0,0,480,481,7,1,0,0,481,482,7,2,0,0, - 482,483,7,0,0,0,483,484,7,1,0,0,484,80,1,0,0,0,485,486,7,1,0,0,486,487, - 7,13,0,0,487,488,7,14,0,0,488,82,1,0,0,0,489,490,7,7,0,0,490,491,7,5,0, - 0,491,492,7,5,0,0,492,493,7,0,0,0,493,494,7,1,0,0,494,495,7,4,0,0,495, - 84,1,0,0,0,496,497,7,5,0,0,497,498,7,1,0,0,498,499,7,4,0,0,499,500,7,3, - 0,0,500,501,7,10,0,0,501,86,1,0,0,0,502,503,7,13,0,0,503,504,7,1,0,0,504, - 505,7,16,0,0,505,506,7,4,0,0,506,88,1,0,0,0,507,508,7,5,0,0,508,509,7, - 17,0,0,509,510,7,6,0,0,510,511,7,0,0,0,511,512,7,4,0,0,512,90,1,0,0,0, - 513,514,7,6,0,0,514,515,7,7,0,0,515,516,7,9,0,0,516,517,7,0,0,0,517,92, - 1,0,0,0,518,519,7,6,0,0,519,520,7,7,0,0,520,521,7,9,0,0,521,94,1,0,0,0, - 522,523,7,7,0,0,523,524,7,13,0,0,524,525,7,2,0,0,525,526,7,22,0,0,526, - 96,1,0,0,0,527,528,7,15,0,0,528,529,7,1,0,0,529,530,7,4,0,0,530,531,7, - 9,0,0,531,532,7,1,0,0,532,533,7,1,0,0,533,534,7,13,0,0,534,98,1,0,0,0, - 535,536,7,20,0,0,536,537,7,13,0,0,537,538,7,17,0,0,538,539,7,7,0,0,539, - 540,7,13,0,0,540,100,1,0,0,0,541,542,7,12,0,0,542,543,7,2,0,0,543,544, - 7,2,0,0,544,102,1,0,0,0,545,546,7,17,0,0,546,547,7,13,0,0,547,548,7,4, - 0,0,548,549,7,1,0,0,549,550,7,6,0,0,550,551,7,0,0,0,551,552,7,1,0,0,552, - 553,7,3,0,0,553,554,7,4,0,0,554,104,1,0,0,0,555,556,7,1,0,0,556,557,7, - 16,0,0,557,558,7,3,0,0,558,559,7,1,0,0,559,560,7,11,0,0,560,561,7,4,0, - 0,561,106,1,0,0,0,562,563,7,3,0,0,563,564,7,6,0,0,564,565,7,1,0,0,565, - 566,7,12,0,0,566,567,7,4,0,0,567,568,7,1,0,0,568,108,1,0,0,0,569,570,7, - 4,0,0,570,571,7,12,0,0,571,572,7,15,0,0,572,573,7,2,0,0,573,574,7,1,0, - 0,574,110,1,0,0,0,575,576,7,15,0,0,576,577,7,1,0,0,577,578,7,19,0,0,578, - 579,7,17,0,0,579,580,7,13,0,0,580,112,1,0,0,0,581,582,7,3,0,0,582,583, - 7,7,0,0,583,584,7,8,0,0,584,585,7,8,0,0,585,586,7,17,0,0,586,587,7,4,0, - 0,587,114,1,0,0,0,588,589,7,6,0,0,589,590,7,7,0,0,590,591,7,2,0,0,591, - 592,7,2,0,0,592,593,7,15,0,0,593,594,7,12,0,0,594,595,7,3,0,0,595,596, - 7,18,0,0,596,116,1,0,0,0,597,598,7,4,0,0,598,599,7,6,0,0,599,600,7,12, - 0,0,600,601,7,13,0,0,601,602,7,0,0,0,602,603,7,12,0,0,603,604,7,3,0,0, - 604,605,7,4,0,0,605,606,7,17,0,0,606,607,7,7,0,0,607,608,7,13,0,0,608, - 118,1,0,0,0,609,610,7,9,0,0,610,611,7,7,0,0,611,612,7,6,0,0,612,613,7, - 18,0,0,613,120,1,0,0,0,614,615,7,12,0,0,615,616,7,2,0,0,616,617,7,4,0, - 0,617,618,7,1,0,0,618,619,7,6,0,0,619,122,1,0,0,0,620,621,7,6,0,0,621, - 622,7,1,0,0,622,623,7,13,0,0,623,624,7,12,0,0,624,625,7,8,0,0,625,626, - 7,1,0,0,626,124,1,0,0,0,627,628,7,4,0,0,628,629,7,7,0,0,629,126,1,0,0, - 0,630,631,7,12,0,0,631,632,7,14,0,0,632,633,7,14,0,0,633,128,1,0,0,0,634, - 635,7,14,0,0,635,636,7,6,0,0,636,637,7,7,0,0,637,638,7,11,0,0,638,130, - 1,0,0,0,639,640,7,3,0,0,640,641,7,7,0,0,641,642,7,2,0,0,642,643,7,20,0, - 0,643,644,7,8,0,0,644,645,7,13,0,0,645,132,1,0,0,0,646,647,7,17,0,0,647, - 648,7,13,0,0,648,649,7,0,0,0,649,650,7,1,0,0,650,651,7,6,0,0,651,652,7, - 4,0,0,652,134,1,0,0,0,653,654,7,17,0,0,654,655,7,13,0,0,655,656,7,4,0, - 0,656,657,7,7,0,0,657,136,1,0,0,0,658,659,7,23,0,0,659,660,7,12,0,0,660, - 661,7,2,0,0,661,662,7,20,0,0,662,663,7,1,0,0,663,664,7,0,0,0,664,138,1, - 0,0,0,665,666,7,11,0,0,666,667,7,6,0,0,667,668,7,17,0,0,668,669,7,8,0, - 0,669,670,7,12,0,0,670,671,7,6,0,0,671,672,7,22,0,0,672,140,1,0,0,0,673, - 674,7,18,0,0,674,675,7,1,0,0,675,676,7,22,0,0,676,142,1,0,0,0,677,678, - 7,3,0,0,678,679,7,7,0,0,679,680,7,13,0,0,680,681,7,0,0,0,681,682,7,4,0, - 0,682,683,7,6,0,0,683,684,7,12,0,0,684,685,7,17,0,0,685,686,7,13,0,0,686, - 687,7,4,0,0,687,144,1,0,0,0,688,689,7,5,0,0,689,690,7,7,0,0,690,691,7, - 6,0,0,691,692,7,1,0,0,692,693,7,17,0,0,693,694,7,19,0,0,694,695,7,13,0, - 0,695,146,1,0,0,0,696,697,7,6,0,0,697,698,7,1,0,0,698,699,7,5,0,0,699, - 700,7,1,0,0,700,701,7,6,0,0,701,702,7,1,0,0,702,703,7,13,0,0,703,704,7, - 3,0,0,704,705,7,1,0,0,705,706,7,0,0,0,706,148,1,0,0,0,707,708,7,14,0,0, - 708,709,7,1,0,0,709,710,7,2,0,0,710,711,7,1,0,0,711,712,7,4,0,0,712,713, - 7,1,0,0,713,150,1,0,0,0,714,715,7,20,0,0,715,716,7,11,0,0,716,717,7,14, - 0,0,717,718,7,12,0,0,718,719,7,4,0,0,719,720,7,1,0,0,720,152,1,0,0,0,721, - 722,7,3,0,0,722,723,7,12,0,0,723,724,7,0,0,0,724,725,7,3,0,0,725,726,7, - 12,0,0,726,727,7,14,0,0,727,728,7,1,0,0,728,154,1,0,0,0,729,730,7,6,0, - 0,730,731,7,1,0,0,731,732,7,0,0,0,732,733,7,4,0,0,733,734,7,6,0,0,734, - 735,7,17,0,0,735,736,7,3,0,0,736,737,7,4,0,0,737,156,1,0,0,0,738,739,7, - 12,0,0,739,740,7,3,0,0,740,741,7,4,0,0,741,742,7,17,0,0,742,743,7,7,0, - 0,743,744,7,13,0,0,744,158,1,0,0,0,745,746,7,0,0,0,746,747,7,1,0,0,747, - 748,7,4,0,0,748,160,1,0,0,0,749,750,7,14,0,0,750,751,7,1,0,0,751,752,7, - 5,0,0,752,753,7,12,0,0,753,754,7,20,0,0,754,755,7,2,0,0,755,756,7,4,0, - 0,756,162,1,0,0,0,757,758,7,13,0,0,758,759,7,7,0,0,759,164,1,0,0,0,760, - 761,7,20,0,0,761,762,7,13,0,0,762,763,7,17,0,0,763,764,7,24,0,0,764,765, - 7,20,0,0,765,766,7,1,0,0,766,166,1,0,0,0,767,768,7,17,0,0,768,769,7,13, - 0,0,769,770,7,14,0,0,770,771,7,1,0,0,771,772,7,16,0,0,772,168,1,0,0,0, - 773,774,7,4,0,0,774,775,7,1,0,0,775,776,7,8,0,0,776,777,7,11,0,0,777,778, - 7,7,0,0,778,779,7,6,0,0,779,780,7,12,0,0,780,781,7,6,0,0,781,782,7,22, - 0,0,782,170,1,0,0,0,783,784,7,9,0,0,784,785,7,17,0,0,785,786,7,4,0,0,786, - 787,7,10,0,0,787,172,1,0,0,0,788,789,7,3,0,0,789,790,7,7,0,0,790,791,7, - 8,0,0,791,792,7,11,0,0,792,793,7,6,0,0,793,794,7,1,0,0,794,795,7,0,0,0, - 795,796,7,0,0,0,796,797,7,17,0,0,797,798,7,7,0,0,798,799,7,13,0,0,799, - 174,1,0,0,0,800,801,7,3,0,0,801,802,7,7,0,0,802,803,7,8,0,0,803,804,7, - 11,0,0,804,176,1,0,0,0,805,806,7,14,0,0,806,807,7,17,0,0,807,808,7,0,0, - 0,808,809,7,12,0,0,809,810,7,2,0,0,810,811,7,2,0,0,811,812,7,7,0,0,812, - 813,7,9,0,0,813,178,1,0,0,0,814,815,7,17,0,0,815,816,7,19,0,0,816,817, - 7,13,0,0,817,818,7,7,0,0,818,819,7,6,0,0,819,820,7,1,0,0,820,180,1,0,0, - 0,821,822,7,3,0,0,822,823,7,10,0,0,823,824,7,1,0,0,824,825,7,3,0,0,825, - 826,7,18,0,0,826,182,1,0,0,0,827,828,7,23,0,0,828,829,7,17,0,0,829,830, - 7,1,0,0,830,831,7,9,0,0,831,184,1,0,0,0,832,833,7,11,0,0,833,834,7,6,0, - 0,834,835,7,7,0,0,835,836,7,3,0,0,836,837,7,1,0,0,837,838,7,14,0,0,838, - 839,7,20,0,0,839,840,7,6,0,0,840,841,7,1,0,0,841,186,1,0,0,0,842,843,7, - 11,0,0,843,844,7,12,0,0,844,845,7,6,0,0,845,846,7,12,0,0,846,847,7,8,0, - 0,847,848,7,1,0,0,848,849,7,4,0,0,849,850,7,1,0,0,850,851,7,6,0,0,851, - 852,7,0,0,0,852,188,1,0,0,0,853,854,7,1,0,0,854,855,7,16,0,0,855,856,7, - 1,0,0,856,857,7,3,0,0,857,858,7,20,0,0,858,859,7,4,0,0,859,860,7,1,0,0, - 860,190,1,0,0,0,861,862,7,1,0,0,862,863,7,16,0,0,863,864,7,1,0,0,864,865, - 7,3,0,0,865,192,1,0,0,0,866,867,7,12,0,0,867,868,7,0,0,0,868,869,7,3,0, - 0,869,194,1,0,0,0,870,871,7,14,0,0,871,872,7,1,0,0,872,873,7,0,0,0,873, - 874,7,3,0,0,874,196,1,0,0,0,875,876,7,4,0,0,876,877,7,6,0,0,877,878,7, - 20,0,0,878,879,7,1,0,0,879,198,1,0,0,0,880,881,7,5,0,0,881,882,7,12,0, - 0,882,883,7,2,0,0,883,884,7,0,0,0,884,885,7,1,0,0,885,200,1,0,0,0,886, - 887,7,13,0,0,887,888,7,20,0,0,888,889,7,2,0,0,889,890,7,2,0,0,890,202, - 1,0,0,0,891,892,5,42,0,0,892,204,1,0,0,0,893,894,5,47,0,0,894,206,1,0, - 0,0,895,896,5,92,0,0,896,208,1,0,0,0,897,898,5,94,0,0,898,210,1,0,0,0, - 899,900,5,43,0,0,900,212,1,0,0,0,901,902,5,45,0,0,902,214,1,0,0,0,903, - 904,5,38,0,0,904,216,1,0,0,0,905,906,5,61,0,0,906,218,1,0,0,0,907,908, - 5,60,0,0,908,912,5,62,0,0,909,910,5,33,0,0,910,912,5,61,0,0,911,907,1, - 0,0,0,911,909,1,0,0,0,912,220,1,0,0,0,913,914,5,60,0,0,914,915,5,61,0, - 0,915,222,1,0,0,0,916,917,5,62,0,0,917,918,5,61,0,0,918,224,1,0,0,0,919, - 920,5,60,0,0,920,226,1,0,0,0,921,922,5,62,0,0,922,228,1,0,0,0,923,924, - 5,40,0,0,924,230,1,0,0,0,925,926,5,41,0,0,926,232,1,0,0,0,927,928,5,44, - 0,0,928,234,1,0,0,0,929,930,5,46,0,0,930,236,1,0,0,0,931,932,5,59,0,0, - 932,238,1,0,0,0,933,934,5,64,0,0,934,935,5,64,0,0,935,936,1,0,0,0,936, - 940,7,25,0,0,937,939,7,26,0,0,938,937,1,0,0,0,939,942,1,0,0,0,940,938, - 1,0,0,0,940,941,1,0,0,0,941,240,1,0,0,0,942,940,1,0,0,0,943,953,5,63,0, - 0,944,945,5,64,0,0,945,949,7,25,0,0,946,948,7,26,0,0,947,946,1,0,0,0,948, - 951,1,0,0,0,949,947,1,0,0,0,949,950,1,0,0,0,950,953,1,0,0,0,951,949,1, - 0,0,0,952,943,1,0,0,0,952,944,1,0,0,0,953,242,1,0,0,0,954,955,5,48,0,0, - 955,957,7,16,0,0,956,958,7,27,0,0,957,956,1,0,0,0,958,959,1,0,0,0,959, - 957,1,0,0,0,959,960,1,0,0,0,960,244,1,0,0,0,961,963,7,28,0,0,962,961,1, - 0,0,0,963,964,1,0,0,0,964,962,1,0,0,0,964,965,1,0,0,0,965,246,1,0,0,0, - 966,968,7,28,0,0,967,966,1,0,0,0,968,969,1,0,0,0,969,967,1,0,0,0,969,970, - 1,0,0,0,970,971,1,0,0,0,971,975,5,46,0,0,972,974,7,28,0,0,973,972,1,0, - 0,0,974,977,1,0,0,0,975,973,1,0,0,0,975,976,1,0,0,0,976,979,1,0,0,0,977, - 975,1,0,0,0,978,980,3,249,124,0,979,978,1,0,0,0,979,980,1,0,0,0,980,997, - 1,0,0,0,981,983,5,46,0,0,982,984,7,28,0,0,983,982,1,0,0,0,984,985,1,0, - 0,0,985,983,1,0,0,0,985,986,1,0,0,0,986,988,1,0,0,0,987,989,3,249,124, - 0,988,987,1,0,0,0,988,989,1,0,0,0,989,997,1,0,0,0,990,992,7,28,0,0,991, - 990,1,0,0,0,992,993,1,0,0,0,993,991,1,0,0,0,993,994,1,0,0,0,994,995,1, - 0,0,0,995,997,3,249,124,0,996,967,1,0,0,0,996,981,1,0,0,0,996,991,1,0, - 0,0,997,248,1,0,0,0,998,1000,7,1,0,0,999,1001,7,29,0,0,1000,999,1,0,0, - 0,1000,1001,1,0,0,0,1001,1003,1,0,0,0,1002,1004,7,28,0,0,1003,1002,1,0, - 0,0,1004,1005,1,0,0,0,1005,1003,1,0,0,0,1005,1006,1,0,0,0,1006,250,1,0, - 0,0,1007,1013,5,34,0,0,1008,1012,8,30,0,0,1009,1010,5,34,0,0,1010,1012, - 5,34,0,0,1011,1008,1,0,0,0,1011,1009,1,0,0,0,1012,1015,1,0,0,0,1013,1011, - 1,0,0,0,1013,1014,1,0,0,0,1014,1016,1,0,0,0,1015,1013,1,0,0,0,1016,1028, - 5,34,0,0,1017,1023,5,39,0,0,1018,1022,8,31,0,0,1019,1020,5,39,0,0,1020, - 1022,5,39,0,0,1021,1018,1,0,0,0,1021,1019,1,0,0,0,1022,1025,1,0,0,0,1023, - 1021,1,0,0,0,1023,1024,1,0,0,0,1024,1026,1,0,0,0,1025,1023,1,0,0,0,1026, - 1028,5,39,0,0,1027,1007,1,0,0,0,1027,1017,1,0,0,0,1028,252,1,0,0,0,1029, - 1033,5,35,0,0,1030,1032,8,32,0,0,1031,1030,1,0,0,0,1032,1035,1,0,0,0,1033, - 1031,1,0,0,0,1033,1034,1,0,0,0,1034,1036,1,0,0,0,1035,1033,1,0,0,0,1036, - 1037,5,35,0,0,1037,254,1,0,0,0,1038,1040,5,123,0,0,1039,1041,3,257,128, - 0,1040,1039,1,0,0,0,1041,1042,1,0,0,0,1042,1040,1,0,0,0,1042,1043,1,0, - 0,0,1043,1044,1,0,0,0,1044,1046,5,45,0,0,1045,1047,3,257,128,0,1046,1045, - 1,0,0,0,1047,1048,1,0,0,0,1048,1046,1,0,0,0,1048,1049,1,0,0,0,1049,1050, - 1,0,0,0,1050,1052,5,45,0,0,1051,1053,3,257,128,0,1052,1051,1,0,0,0,1053, - 1054,1,0,0,0,1054,1052,1,0,0,0,1054,1055,1,0,0,0,1055,1056,1,0,0,0,1056, - 1058,5,45,0,0,1057,1059,3,257,128,0,1058,1057,1,0,0,0,1059,1060,1,0,0, - 0,1060,1058,1,0,0,0,1060,1061,1,0,0,0,1061,1062,1,0,0,0,1062,1064,5,45, - 0,0,1063,1065,3,257,128,0,1064,1063,1,0,0,0,1065,1066,1,0,0,0,1066,1064, - 1,0,0,0,1066,1067,1,0,0,0,1067,1068,1,0,0,0,1068,1069,5,125,0,0,1069,256, - 1,0,0,0,1070,1071,7,27,0,0,1071,258,1,0,0,0,1072,1074,5,91,0,0,1073,1075, - 8,33,0,0,1074,1073,1,0,0,0,1075,1076,1,0,0,0,1076,1074,1,0,0,0,1076,1077, - 1,0,0,0,1077,1078,1,0,0,0,1078,1079,5,93,0,0,1079,260,1,0,0,0,1080,1082, - 5,96,0,0,1081,1083,8,34,0,0,1082,1081,1,0,0,0,1083,1084,1,0,0,0,1084,1082, - 1,0,0,0,1084,1085,1,0,0,0,1085,1086,1,0,0,0,1086,1087,5,96,0,0,1087,262, - 1,0,0,0,1088,1092,7,25,0,0,1089,1091,7,26,0,0,1090,1089,1,0,0,0,1091,1094, - 1,0,0,0,1092,1090,1,0,0,0,1092,1093,1,0,0,0,1093,1096,1,0,0,0,1094,1092, - 1,0,0,0,1095,1097,5,36,0,0,1096,1095,1,0,0,0,1096,1097,1,0,0,0,1097,264, - 1,0,0,0,1098,1100,7,35,0,0,1099,1098,1,0,0,0,1100,1101,1,0,0,0,1101,1099, - 1,0,0,0,1101,1102,1,0,0,0,1102,1103,1,0,0,0,1103,1104,6,132,0,0,1104,266, - 1,0,0,0,1105,1106,5,45,0,0,1106,1107,5,45,0,0,1107,1111,1,0,0,0,1108,1110, - 8,36,0,0,1109,1108,1,0,0,0,1110,1113,1,0,0,0,1111,1109,1,0,0,0,1111,1112, - 1,0,0,0,1112,1114,1,0,0,0,1113,1111,1,0,0,0,1114,1115,6,133,0,0,1115,268, - 1,0,0,0,1116,1117,5,47,0,0,1117,1118,5,42,0,0,1118,1122,1,0,0,0,1119,1121, - 9,0,0,0,1120,1119,1,0,0,0,1121,1124,1,0,0,0,1122,1123,1,0,0,0,1122,1120, - 1,0,0,0,1123,1125,1,0,0,0,1124,1122,1,0,0,0,1125,1126,5,42,0,0,1126,1127, - 5,47,0,0,1127,1128,1,0,0,0,1128,1129,6,134,0,0,1129,270,1,0,0,0,34,0,911, - 940,949,952,959,964,969,975,979,985,988,993,996,1000,1005,1011,1013,1021, - 1023,1027,1033,1042,1048,1054,1060,1066,1076,1084,1092,1096,1101,1111, - 1122,1,6,0,0 + 7,134,2,135,7,135,2,136,7,136,2,137,7,137,2,138,7,138,2,139,7,139,2,140, + 7,140,2,141,7,141,2,142,7,142,2,143,7,143,2,144,7,144,2,145,7,145,2,146, + 7,146,2,147,7,147,2,148,7,148,2,149,7,149,2,150,7,150,2,151,7,151,2,152, + 7,152,2,153,7,153,2,154,7,154,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,1,1,1,1,1, + 1,1,1,1,1,2,1,2,1,2,1,2,1,2,1,2,1,3,1,3,1,3,1,3,1,4,1,4,1,4,1,5,1,5,1, + 5,1,5,1,6,1,6,1,6,1,7,1,7,1,7,1,7,1,8,1,8,1,8,1,8,1,9,1,9,1,9,1,9,1,10, + 1,10,1,10,1,10,1,11,1,11,1,11,1,11,1,11,1,12,1,12,1,12,1,12,1,13,1,13, + 1,13,1,13,1,13,1,14,1,14,1,14,1,14,1,14,1,15,1,15,1,15,1,15,1,15,1,16, + 1,16,1,16,1,16,1,17,1,17,1,17,1,17,1,17,1,17,1,18,1,18,1,18,1,18,1,18, + 1,19,1,19,1,19,1,19,1,19,1,19,1,20,1,20,1,20,1,20,1,20,1,21,1,21,1,21, + 1,21,1,21,1,21,1,22,1,22,1,22,1,22,1,22,1,23,1,23,1,23,1,24,1,24,1,24, + 1,25,1,25,1,25,1,25,1,25,1,25,1,26,1,26,1,26,1,26,1,26,1,26,1,27,1,27, + 1,27,1,28,1,28,1,28,1,29,1,29,1,29,1,29,1,29,1,29,1,29,1,30,1,30,1,30, + 1,30,1,30,1,30,1,30,1,31,1,31,1,31,1,32,1,32,1,32,1,32,1,32,1,33,1,33, + 1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,34,1,34,1,34,1,34, + 1,34,1,34,1,34,1,34,1,34,1,35,1,35,1,35,1,35,1,35,1,35,1,35,1,35,1,36, + 1,36,1,36,1,36,1,36,1,36,1,37,1,37,1,37,1,37,1,37,1,37,1,38,1,38,1,38, + 1,38,1,38,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,40,1,40, + 1,40,1,40,1,40,1,41,1,41,1,41,1,41,1,41,1,42,1,42,1,42,1,42,1,42,1,43, + 1,43,1,43,1,43,1,44,1,44,1,44,1,44,1,44,1,44,1,44,1,45,1,45,1,45,1,45, + 1,45,1,45,1,46,1,46,1,46,1,46,1,46,1,47,1,47,1,47,1,47,1,47,1,47,1,48, + 1,48,1,48,1,48,1,48,1,49,1,49,1,49,1,49,1,50,1,50,1,50,1,50,1,50,1,51, + 1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,52,1,52,1,52,1,52,1,52,1,52,1,53, + 1,53,1,53,1,53,1,54,1,54,1,54,1,54,1,54,1,54,1,54,1,54,1,54,1,54,1,55, + 1,55,1,55,1,55,1,55,1,55,1,55,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,57, + 1,57,1,57,1,57,1,57,1,57,1,58,1,58,1,58,1,58,1,58,1,58,1,59,1,59,1,59, + 1,59,1,59,1,59,1,59,1,60,1,60,1,60,1,60,1,60,1,60,1,60,1,60,1,60,1,61, + 1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,61,1,62,1,62,1,62, + 1,62,1,62,1,63,1,63,1,63,1,63,1,63,1,63,1,64,1,64,1,64,1,64,1,64,1,64, + 1,64,1,65,1,65,1,65,1,66,1,66,1,66,1,66,1,67,1,67,1,67,1,67,1,67,1,68, + 1,68,1,68,1,68,1,68,1,68,1,68,1,69,1,69,1,69,1,69,1,69,1,69,1,69,1,70, + 1,70,1,70,1,70,1,70,1,71,1,71,1,71,1,71,1,71,1,71,1,71,1,72,1,72,1,72, + 1,72,1,72,1,72,1,72,1,72,1,73,1,73,1,73,1,73,1,74,1,74,1,74,1,74,1,74, + 1,74,1,74,1,74,1,74,1,74,1,74,1,75,1,75,1,75,1,75,1,75,1,75,1,75,1,75, + 1,76,1,76,1,76,1,76,1,76,1,76,1,76,1,76,1,76,1,76,1,76,1,77,1,77,1,77, + 1,77,1,77,1,77,1,77,1,78,1,78,1,78,1,78,1,78,1,78,1,78,1,79,1,79,1,79, + 1,79,1,79,1,79,1,79,1,79,1,80,1,80,1,80,1,80,1,80,1,80,1,80,1,80,1,80, + 1,81,1,81,1,81,1,81,1,81,1,81,1,81,1,82,1,82,1,82,1,82,1,83,1,83,1,83, + 1,83,1,83,1,83,1,83,1,83,1,84,1,84,1,84,1,85,1,85,1,85,1,85,1,85,1,85, + 1,85,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,86,1,87,1,87,1,87, + 1,87,1,87,1,87,1,87,1,87,1,87,1,88,1,88,1,88,1,88,1,88,1,88,1,88,1,88, + 1,88,1,88,1,88,1,88,1,88,1,89,1,89,1,89,1,89,1,89,1,89,1,90,1,90,1,90, + 1,90,1,90,1,90,1,90,1,90,1,90,1,90,1,91,1,91,1,91,1,91,1,91,1,92,1,92, + 1,92,1,92,1,92,1,92,1,92,1,92,1,92,1,92,1,92,1,92,1,93,1,93,1,93,1,93, + 1,93,1,94,1,94,1,94,1,94,1,94,1,94,1,94,1,94,1,94,1,95,1,95,1,95,1,95, + 1,95,1,95,1,95,1,96,1,96,1,96,1,96,1,96,1,96,1,97,1,97,1,97,1,97,1,97, + 1,98,1,98,1,98,1,98,1,98,1,98,1,98,1,98,1,98,1,98,1,99,1,99,1,99,1,99, + 1,99,1,99,1,99,1,99,1,99,1,99,1,99,1,100,1,100,1,100,1,100,1,100,1,100, + 1,100,1,100,1,101,1,101,1,101,1,101,1,101,1,102,1,102,1,102,1,102,1,103, + 1,103,1,103,1,103,1,103,1,104,1,104,1,104,1,104,1,104,1,105,1,105,1,105, + 1,105,1,105,1,105,1,106,1,106,1,106,1,106,1,106,1,107,1,107,1,107,1,107, + 1,107,1,107,1,108,1,108,1,108,1,108,1,108,1,108,1,108,1,109,1,109,1,109, + 1,109,1,109,1,109,1,109,1,109,1,109,1,109,1,110,1,110,1,110,1,110,1,110, + 1,110,1,110,1,110,1,110,1,110,1,111,1,111,1,111,1,111,1,111,1,111,1,111, + 1,111,1,111,1,111,1,112,1,112,1,112,1,112,1,112,1,112,1,112,1,112,1,113, + 1,113,1,113,1,113,1,113,1,113,1,113,1,113,1,114,1,114,1,114,1,114,1,114, + 1,115,1,115,1,115,1,115,1,115,1,115,1,115,1,116,1,116,1,116,1,116,1,116, + 1,116,1,116,1,117,1,117,1,117,1,117,1,117,1,118,1,118,1,118,1,118,1,118, + 1,118,1,118,1,118,1,119,1,119,1,119,1,119,1,119,1,119,1,120,1,120,1,120, + 1,120,1,120,1,120,1,120,1,121,1,121,1,122,1,122,1,123,1,123,1,124,1,124, + 1,125,1,125,1,126,1,126,1,127,1,127,1,128,1,128,1,129,1,129,1,129,1,129, + 3,129,1100,8,129,1,130,1,130,1,130,1,131,1,131,1,131,1,132,1,132,1,133, + 1,133,1,134,1,134,1,135,1,135,1,136,1,136,1,137,1,137,1,138,1,138,1,139, + 1,139,1,139,1,139,1,139,5,139,1127,8,139,10,139,12,139,1130,9,139,1,140, + 1,140,1,140,1,140,5,140,1136,8,140,10,140,12,140,1139,9,140,3,140,1141, + 8,140,1,141,1,141,1,141,4,141,1146,8,141,11,141,12,141,1147,1,142,4,142, + 1151,8,142,11,142,12,142,1152,1,143,4,143,1156,8,143,11,143,12,143,1157, + 1,143,1,143,5,143,1162,8,143,10,143,12,143,1165,9,143,1,143,3,143,1168, + 8,143,1,143,1,143,4,143,1172,8,143,11,143,12,143,1173,1,143,3,143,1177, + 8,143,1,143,4,143,1180,8,143,11,143,12,143,1181,1,143,3,143,1185,8,143, + 1,144,1,144,3,144,1189,8,144,1,144,4,144,1192,8,144,11,144,12,144,1193, + 1,145,1,145,1,145,1,145,5,145,1200,8,145,10,145,12,145,1203,9,145,1,145, + 1,145,1,145,1,145,1,145,5,145,1210,8,145,10,145,12,145,1213,9,145,1,145, + 3,145,1216,8,145,1,146,1,146,5,146,1220,8,146,10,146,12,146,1223,9,146, + 1,146,1,146,1,147,1,147,4,147,1229,8,147,11,147,12,147,1230,1,147,1,147, + 4,147,1235,8,147,11,147,12,147,1236,1,147,1,147,4,147,1241,8,147,11,147, + 12,147,1242,1,147,1,147,4,147,1247,8,147,11,147,12,147,1248,1,147,1,147, + 4,147,1253,8,147,11,147,12,147,1254,1,147,1,147,1,148,1,148,1,149,1,149, + 4,149,1263,8,149,11,149,12,149,1264,1,149,1,149,1,150,1,150,4,150,1271, + 8,150,11,150,12,150,1272,1,150,1,150,1,151,1,151,5,151,1279,8,151,10,151, + 12,151,1282,9,151,1,151,3,151,1285,8,151,1,152,4,152,1288,8,152,11,152, + 12,152,1289,1,152,1,152,1,153,1,153,1,153,1,153,1,153,5,153,1299,8,153, + 10,153,12,153,1302,9,153,1,153,1,153,1,154,1,154,1,154,1,154,5,154,1310, + 8,154,10,154,12,154,1313,9,154,1,154,1,154,1,154,1,154,1,154,1,1311,0, + 155,1,1,3,2,5,3,7,4,9,5,11,6,13,7,15,8,17,9,19,10,21,11,23,12,25,13,27, + 14,29,15,31,16,33,17,35,18,37,19,39,20,41,21,43,22,45,23,47,24,49,25,51, + 26,53,27,55,28,57,29,59,30,61,31,63,32,65,33,67,34,69,35,71,36,73,37,75, + 38,77,39,79,40,81,41,83,42,85,43,87,44,89,45,91,46,93,47,95,48,97,49,99, + 50,101,51,103,52,105,53,107,54,109,55,111,56,113,57,115,58,117,59,119, + 60,121,61,123,62,125,63,127,64,129,65,131,66,133,67,135,68,137,69,139, + 70,141,71,143,72,145,73,147,74,149,75,151,76,153,77,155,78,157,79,159, + 80,161,81,163,82,165,83,167,84,169,85,171,86,173,87,175,88,177,89,179, + 90,181,91,183,92,185,93,187,94,189,95,191,96,193,97,195,98,197,99,199, + 100,201,101,203,102,205,103,207,104,209,105,211,106,213,107,215,108,217, + 109,219,110,221,111,223,112,225,113,227,114,229,115,231,116,233,117,235, + 118,237,119,239,120,241,121,243,122,245,123,247,124,249,125,251,126,253, + 127,255,128,257,129,259,130,261,131,263,132,265,133,267,134,269,135,271, + 136,273,137,275,138,277,139,279,140,281,141,283,142,285,143,287,144,289, + 0,291,145,293,146,295,147,297,0,299,148,301,149,303,150,305,151,307,152, + 309,153,1,0,37,2,0,83,83,115,115,2,0,69,69,101,101,2,0,76,76,108,108,2, + 0,67,67,99,99,2,0,84,84,116,116,2,0,70,70,102,102,2,0,82,82,114,114,2, + 0,79,79,111,111,2,0,77,77,109,109,2,0,87,87,119,119,2,0,72,72,104,104, + 2,0,80,80,112,112,2,0,65,65,97,97,2,0,78,78,110,110,2,0,68,68,100,100, + 2,0,88,88,120,120,2,0,81,81,113,113,2,0,86,86,118,118,2,0,73,73,105,105, + 2,0,66,66,98,98,2,0,75,75,107,107,2,0,71,71,103,103,2,0,85,85,117,117, + 2,0,74,74,106,106,2,0,89,89,121,121,3,0,65,90,95,95,97,122,4,0,48,57,65, + 90,95,95,97,122,3,0,48,57,65,70,97,102,1,0,48,57,2,0,43,43,45,45,1,0,34, + 34,1,0,39,39,1,0,35,35,1,0,93,93,1,0,96,96,3,0,9,10,13,13,32,32,2,0,10, + 10,13,13,1350,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,9,1,0, + 0,0,0,11,1,0,0,0,0,13,1,0,0,0,0,15,1,0,0,0,0,17,1,0,0,0,0,19,1,0,0,0,0, + 21,1,0,0,0,0,23,1,0,0,0,0,25,1,0,0,0,0,27,1,0,0,0,0,29,1,0,0,0,0,31,1, + 0,0,0,0,33,1,0,0,0,0,35,1,0,0,0,0,37,1,0,0,0,0,39,1,0,0,0,0,41,1,0,0,0, + 0,43,1,0,0,0,0,45,1,0,0,0,0,47,1,0,0,0,0,49,1,0,0,0,0,51,1,0,0,0,0,53, + 1,0,0,0,0,55,1,0,0,0,0,57,1,0,0,0,0,59,1,0,0,0,0,61,1,0,0,0,0,63,1,0,0, + 0,0,65,1,0,0,0,0,67,1,0,0,0,0,69,1,0,0,0,0,71,1,0,0,0,0,73,1,0,0,0,0,75, + 1,0,0,0,0,77,1,0,0,0,0,79,1,0,0,0,0,81,1,0,0,0,0,83,1,0,0,0,0,85,1,0,0, + 0,0,87,1,0,0,0,0,89,1,0,0,0,0,91,1,0,0,0,0,93,1,0,0,0,0,95,1,0,0,0,0,97, + 1,0,0,0,0,99,1,0,0,0,0,101,1,0,0,0,0,103,1,0,0,0,0,105,1,0,0,0,0,107,1, + 0,0,0,0,109,1,0,0,0,0,111,1,0,0,0,0,113,1,0,0,0,0,115,1,0,0,0,0,117,1, + 0,0,0,0,119,1,0,0,0,0,121,1,0,0,0,0,123,1,0,0,0,0,125,1,0,0,0,0,127,1, + 0,0,0,0,129,1,0,0,0,0,131,1,0,0,0,0,133,1,0,0,0,0,135,1,0,0,0,0,137,1, + 0,0,0,0,139,1,0,0,0,0,141,1,0,0,0,0,143,1,0,0,0,0,145,1,0,0,0,0,147,1, + 0,0,0,0,149,1,0,0,0,0,151,1,0,0,0,0,153,1,0,0,0,0,155,1,0,0,0,0,157,1, + 0,0,0,0,159,1,0,0,0,0,161,1,0,0,0,0,163,1,0,0,0,0,165,1,0,0,0,0,167,1, + 0,0,0,0,169,1,0,0,0,0,171,1,0,0,0,0,173,1,0,0,0,0,175,1,0,0,0,0,177,1, + 0,0,0,0,179,1,0,0,0,0,181,1,0,0,0,0,183,1,0,0,0,0,185,1,0,0,0,0,187,1, + 0,0,0,0,189,1,0,0,0,0,191,1,0,0,0,0,193,1,0,0,0,0,195,1,0,0,0,0,197,1, + 0,0,0,0,199,1,0,0,0,0,201,1,0,0,0,0,203,1,0,0,0,0,205,1,0,0,0,0,207,1, + 0,0,0,0,209,1,0,0,0,0,211,1,0,0,0,0,213,1,0,0,0,0,215,1,0,0,0,0,217,1, + 0,0,0,0,219,1,0,0,0,0,221,1,0,0,0,0,223,1,0,0,0,0,225,1,0,0,0,0,227,1, + 0,0,0,0,229,1,0,0,0,0,231,1,0,0,0,0,233,1,0,0,0,0,235,1,0,0,0,0,237,1, + 0,0,0,0,239,1,0,0,0,0,241,1,0,0,0,0,243,1,0,0,0,0,245,1,0,0,0,0,247,1, + 0,0,0,0,249,1,0,0,0,0,251,1,0,0,0,0,253,1,0,0,0,0,255,1,0,0,0,0,257,1, + 0,0,0,0,259,1,0,0,0,0,261,1,0,0,0,0,263,1,0,0,0,0,265,1,0,0,0,0,267,1, + 0,0,0,0,269,1,0,0,0,0,271,1,0,0,0,0,273,1,0,0,0,0,275,1,0,0,0,0,277,1, + 0,0,0,0,279,1,0,0,0,0,281,1,0,0,0,0,283,1,0,0,0,0,285,1,0,0,0,0,287,1, + 0,0,0,0,291,1,0,0,0,0,293,1,0,0,0,0,295,1,0,0,0,0,299,1,0,0,0,0,301,1, + 0,0,0,0,303,1,0,0,0,0,305,1,0,0,0,0,307,1,0,0,0,0,309,1,0,0,0,1,311,1, + 0,0,0,3,318,1,0,0,0,5,323,1,0,0,0,7,329,1,0,0,0,9,333,1,0,0,0,11,336,1, + 0,0,0,13,340,1,0,0,0,15,343,1,0,0,0,17,347,1,0,0,0,19,351,1,0,0,0,21,355, + 1,0,0,0,23,359,1,0,0,0,25,364,1,0,0,0,27,368,1,0,0,0,29,373,1,0,0,0,31, + 378,1,0,0,0,33,383,1,0,0,0,35,387,1,0,0,0,37,393,1,0,0,0,39,398,1,0,0, + 0,41,404,1,0,0,0,43,409,1,0,0,0,45,415,1,0,0,0,47,420,1,0,0,0,49,423,1, + 0,0,0,51,426,1,0,0,0,53,432,1,0,0,0,55,438,1,0,0,0,57,441,1,0,0,0,59,444, + 1,0,0,0,61,451,1,0,0,0,63,458,1,0,0,0,65,461,1,0,0,0,67,466,1,0,0,0,69, + 478,1,0,0,0,71,487,1,0,0,0,73,495,1,0,0,0,75,501,1,0,0,0,77,507,1,0,0, + 0,79,512,1,0,0,0,81,522,1,0,0,0,83,527,1,0,0,0,85,532,1,0,0,0,87,537,1, + 0,0,0,89,541,1,0,0,0,91,548,1,0,0,0,93,554,1,0,0,0,95,559,1,0,0,0,97,565, + 1,0,0,0,99,570,1,0,0,0,101,574,1,0,0,0,103,579,1,0,0,0,105,587,1,0,0,0, + 107,593,1,0,0,0,109,597,1,0,0,0,111,607,1,0,0,0,113,614,1,0,0,0,115,621, + 1,0,0,0,117,627,1,0,0,0,119,633,1,0,0,0,121,640,1,0,0,0,123,649,1,0,0, + 0,125,661,1,0,0,0,127,666,1,0,0,0,129,672,1,0,0,0,131,679,1,0,0,0,133, + 682,1,0,0,0,135,686,1,0,0,0,137,691,1,0,0,0,139,698,1,0,0,0,141,705,1, + 0,0,0,143,710,1,0,0,0,145,717,1,0,0,0,147,725,1,0,0,0,149,729,1,0,0,0, + 151,740,1,0,0,0,153,748,1,0,0,0,155,759,1,0,0,0,157,766,1,0,0,0,159,773, + 1,0,0,0,161,781,1,0,0,0,163,790,1,0,0,0,165,797,1,0,0,0,167,801,1,0,0, + 0,169,809,1,0,0,0,171,812,1,0,0,0,173,819,1,0,0,0,175,829,1,0,0,0,177, + 838,1,0,0,0,179,851,1,0,0,0,181,857,1,0,0,0,183,867,1,0,0,0,185,872,1, + 0,0,0,187,884,1,0,0,0,189,889,1,0,0,0,191,898,1,0,0,0,193,905,1,0,0,0, + 195,911,1,0,0,0,197,916,1,0,0,0,199,926,1,0,0,0,201,937,1,0,0,0,203,945, + 1,0,0,0,205,950,1,0,0,0,207,954,1,0,0,0,209,959,1,0,0,0,211,964,1,0,0, + 0,213,970,1,0,0,0,215,975,1,0,0,0,217,981,1,0,0,0,219,988,1,0,0,0,221, + 998,1,0,0,0,223,1008,1,0,0,0,225,1018,1,0,0,0,227,1026,1,0,0,0,229,1034, + 1,0,0,0,231,1039,1,0,0,0,233,1046,1,0,0,0,235,1053,1,0,0,0,237,1058,1, + 0,0,0,239,1066,1,0,0,0,241,1072,1,0,0,0,243,1079,1,0,0,0,245,1081,1,0, + 0,0,247,1083,1,0,0,0,249,1085,1,0,0,0,251,1087,1,0,0,0,253,1089,1,0,0, + 0,255,1091,1,0,0,0,257,1093,1,0,0,0,259,1099,1,0,0,0,261,1101,1,0,0,0, + 263,1104,1,0,0,0,265,1107,1,0,0,0,267,1109,1,0,0,0,269,1111,1,0,0,0,271, + 1113,1,0,0,0,273,1115,1,0,0,0,275,1117,1,0,0,0,277,1119,1,0,0,0,279,1121, + 1,0,0,0,281,1140,1,0,0,0,283,1142,1,0,0,0,285,1150,1,0,0,0,287,1184,1, + 0,0,0,289,1186,1,0,0,0,291,1215,1,0,0,0,293,1217,1,0,0,0,295,1226,1,0, + 0,0,297,1258,1,0,0,0,299,1260,1,0,0,0,301,1268,1,0,0,0,303,1276,1,0,0, + 0,305,1287,1,0,0,0,307,1293,1,0,0,0,309,1305,1,0,0,0,311,312,7,0,0,0,312, + 313,7,1,0,0,313,314,7,2,0,0,314,315,7,1,0,0,315,316,7,3,0,0,316,317,7, + 4,0,0,317,2,1,0,0,0,318,319,7,5,0,0,319,320,7,6,0,0,320,321,7,7,0,0,321, + 322,7,8,0,0,322,4,1,0,0,0,323,324,7,9,0,0,324,325,7,10,0,0,325,326,7,1, + 0,0,326,327,7,6,0,0,327,328,7,1,0,0,328,6,1,0,0,0,329,330,7,4,0,0,330, + 331,7,7,0,0,331,332,7,11,0,0,332,8,1,0,0,0,333,334,7,12,0,0,334,335,7, + 0,0,0,335,10,1,0,0,0,336,337,7,12,0,0,337,338,7,13,0,0,338,339,7,14,0, + 0,339,12,1,0,0,0,340,341,7,7,0,0,341,342,7,6,0,0,342,14,1,0,0,0,343,344, + 7,13,0,0,344,345,7,7,0,0,345,346,7,4,0,0,346,16,1,0,0,0,347,348,7,15,0, + 0,348,349,7,7,0,0,349,350,7,6,0,0,350,18,1,0,0,0,351,352,7,1,0,0,352,353, + 7,16,0,0,353,354,7,17,0,0,354,20,1,0,0,0,355,356,7,18,0,0,356,357,7,8, + 0,0,357,358,7,11,0,0,358,22,1,0,0,0,359,360,7,19,0,0,360,361,7,12,0,0, + 361,362,7,13,0,0,362,363,7,14,0,0,363,24,1,0,0,0,364,365,7,19,0,0,365, + 366,7,7,0,0,366,367,7,6,0,0,367,26,1,0,0,0,368,369,7,19,0,0,369,370,7, + 15,0,0,370,371,7,7,0,0,371,372,7,6,0,0,372,28,1,0,0,0,373,374,7,19,0,0, + 374,375,7,13,0,0,375,376,7,7,0,0,376,377,7,4,0,0,377,30,1,0,0,0,378,379, + 7,2,0,0,379,380,7,18,0,0,380,381,7,20,0,0,381,382,7,1,0,0,382,32,1,0,0, + 0,383,384,7,8,0,0,384,385,7,7,0,0,385,386,7,14,0,0,386,34,1,0,0,0,387, + 388,7,18,0,0,388,389,7,13,0,0,389,390,7,13,0,0,390,391,7,1,0,0,391,392, + 7,6,0,0,392,36,1,0,0,0,393,394,7,2,0,0,394,395,7,1,0,0,395,396,7,5,0,0, + 396,397,7,4,0,0,397,38,1,0,0,0,398,399,7,6,0,0,399,400,7,18,0,0,400,401, + 7,21,0,0,401,402,7,10,0,0,402,403,7,4,0,0,403,40,1,0,0,0,404,405,7,5,0, + 0,405,406,7,22,0,0,406,407,7,2,0,0,407,408,7,2,0,0,408,42,1,0,0,0,409, + 410,7,7,0,0,410,411,7,22,0,0,411,412,7,4,0,0,412,413,7,1,0,0,413,414,7, + 6,0,0,414,44,1,0,0,0,415,416,7,23,0,0,416,417,7,7,0,0,417,418,7,18,0,0, + 418,419,7,13,0,0,419,46,1,0,0,0,420,421,7,18,0,0,421,422,7,13,0,0,422, + 48,1,0,0,0,423,424,7,7,0,0,424,425,7,13,0,0,425,50,1,0,0,0,426,427,7,7, + 0,0,427,428,7,6,0,0,428,429,7,14,0,0,429,430,7,1,0,0,430,431,7,6,0,0,431, + 52,1,0,0,0,432,433,7,21,0,0,433,434,7,6,0,0,434,435,7,7,0,0,435,436,7, + 22,0,0,436,437,7,11,0,0,437,54,1,0,0,0,438,439,7,18,0,0,439,440,7,0,0, + 0,440,56,1,0,0,0,441,442,7,19,0,0,442,443,7,24,0,0,443,58,1,0,0,0,444, + 445,7,10,0,0,445,446,7,12,0,0,446,447,7,17,0,0,447,448,7,18,0,0,448,449, + 7,13,0,0,449,450,7,21,0,0,450,60,1,0,0,0,451,452,7,1,0,0,452,453,7,15, + 0,0,453,454,7,18,0,0,454,455,7,0,0,0,455,456,7,4,0,0,456,457,7,0,0,0,457, + 62,1,0,0,0,458,459,7,18,0,0,459,460,7,5,0,0,460,64,1,0,0,0,461,462,7,4, + 0,0,462,463,7,10,0,0,463,464,7,1,0,0,464,465,7,13,0,0,465,66,1,0,0,0,466, + 467,7,14,0,0,467,468,7,18,0,0,468,469,7,0,0,0,469,470,7,4,0,0,470,471, + 7,18,0,0,471,472,7,13,0,0,472,473,7,3,0,0,473,474,7,4,0,0,474,475,7,6, + 0,0,475,476,7,7,0,0,476,477,7,9,0,0,477,68,1,0,0,0,478,479,7,14,0,0,479, + 480,7,18,0,0,480,481,7,0,0,0,481,482,7,4,0,0,482,483,7,18,0,0,483,484, + 7,13,0,0,484,485,7,3,0,0,485,486,7,4,0,0,486,70,1,0,0,0,487,488,7,11,0, + 0,488,489,7,1,0,0,489,490,7,6,0,0,490,491,7,3,0,0,491,492,7,1,0,0,492, + 493,7,13,0,0,493,494,7,4,0,0,494,72,1,0,0,0,495,496,7,3,0,0,496,497,7, + 6,0,0,497,498,7,7,0,0,498,499,7,0,0,0,499,500,7,0,0,0,500,74,1,0,0,0,501, + 502,7,12,0,0,502,503,7,11,0,0,503,504,7,11,0,0,504,505,7,2,0,0,505,506, + 7,24,0,0,506,76,1,0,0,0,507,508,7,7,0,0,508,509,7,17,0,0,509,510,7,1,0, + 0,510,511,7,6,0,0,511,78,1,0,0,0,512,513,7,11,0,0,513,514,7,12,0,0,514, + 515,7,6,0,0,515,516,7,4,0,0,516,517,7,18,0,0,517,518,7,4,0,0,518,519,7, + 18,0,0,519,520,7,7,0,0,520,521,7,13,0,0,521,80,1,0,0,0,522,523,7,3,0,0, + 523,524,7,12,0,0,524,525,7,0,0,0,525,526,7,1,0,0,526,82,1,0,0,0,527,528, + 7,9,0,0,528,529,7,10,0,0,529,530,7,1,0,0,530,531,7,13,0,0,531,84,1,0,0, + 0,532,533,7,1,0,0,533,534,7,2,0,0,534,535,7,0,0,0,535,536,7,1,0,0,536, + 86,1,0,0,0,537,538,7,1,0,0,538,539,7,13,0,0,539,540,7,14,0,0,540,88,1, + 0,0,0,541,542,7,7,0,0,542,543,7,5,0,0,543,544,7,5,0,0,544,545,7,0,0,0, + 545,546,7,1,0,0,546,547,7,4,0,0,547,90,1,0,0,0,548,549,7,5,0,0,549,550, + 7,1,0,0,550,551,7,4,0,0,551,552,7,3,0,0,552,553,7,10,0,0,553,92,1,0,0, + 0,554,555,7,13,0,0,555,556,7,1,0,0,556,557,7,15,0,0,557,558,7,4,0,0,558, + 94,1,0,0,0,559,560,7,5,0,0,560,561,7,18,0,0,561,562,7,6,0,0,562,563,7, + 0,0,0,563,564,7,4,0,0,564,96,1,0,0,0,565,566,7,6,0,0,566,567,7,7,0,0,567, + 568,7,9,0,0,568,569,7,0,0,0,569,98,1,0,0,0,570,571,7,6,0,0,571,572,7,7, + 0,0,572,573,7,9,0,0,573,100,1,0,0,0,574,575,7,7,0,0,575,576,7,13,0,0,576, + 577,7,2,0,0,577,578,7,24,0,0,578,102,1,0,0,0,579,580,7,19,0,0,580,581, + 7,1,0,0,581,582,7,4,0,0,582,583,7,9,0,0,583,584,7,1,0,0,584,585,7,1,0, + 0,585,586,7,13,0,0,586,104,1,0,0,0,587,588,7,22,0,0,588,589,7,13,0,0,589, + 590,7,18,0,0,590,591,7,7,0,0,591,592,7,13,0,0,592,106,1,0,0,0,593,594, + 7,12,0,0,594,595,7,2,0,0,595,596,7,2,0,0,596,108,1,0,0,0,597,598,7,18, + 0,0,598,599,7,13,0,0,599,600,7,4,0,0,600,601,7,1,0,0,601,602,7,6,0,0,602, + 603,7,0,0,0,603,604,7,1,0,0,604,605,7,3,0,0,605,606,7,4,0,0,606,110,1, + 0,0,0,607,608,7,1,0,0,608,609,7,15,0,0,609,610,7,3,0,0,610,611,7,1,0,0, + 611,612,7,11,0,0,612,613,7,4,0,0,613,112,1,0,0,0,614,615,7,3,0,0,615,616, + 7,6,0,0,616,617,7,1,0,0,617,618,7,12,0,0,618,619,7,4,0,0,619,620,7,1,0, + 0,620,114,1,0,0,0,621,622,7,4,0,0,622,623,7,12,0,0,623,624,7,19,0,0,624, + 625,7,2,0,0,625,626,7,1,0,0,626,116,1,0,0,0,627,628,7,19,0,0,628,629,7, + 1,0,0,629,630,7,21,0,0,630,631,7,18,0,0,631,632,7,13,0,0,632,118,1,0,0, + 0,633,634,7,3,0,0,634,635,7,7,0,0,635,636,7,8,0,0,636,637,7,8,0,0,637, + 638,7,18,0,0,638,639,7,4,0,0,639,120,1,0,0,0,640,641,7,6,0,0,641,642,7, + 7,0,0,642,643,7,2,0,0,643,644,7,2,0,0,644,645,7,19,0,0,645,646,7,12,0, + 0,646,647,7,3,0,0,647,648,7,20,0,0,648,122,1,0,0,0,649,650,7,4,0,0,650, + 651,7,6,0,0,651,652,7,12,0,0,652,653,7,13,0,0,653,654,7,0,0,0,654,655, + 7,12,0,0,655,656,7,3,0,0,656,657,7,4,0,0,657,658,7,18,0,0,658,659,7,7, + 0,0,659,660,7,13,0,0,660,124,1,0,0,0,661,662,7,9,0,0,662,663,7,7,0,0,663, + 664,7,6,0,0,664,665,7,20,0,0,665,126,1,0,0,0,666,667,7,12,0,0,667,668, + 7,2,0,0,668,669,7,4,0,0,669,670,7,1,0,0,670,671,7,6,0,0,671,128,1,0,0, + 0,672,673,7,6,0,0,673,674,7,1,0,0,674,675,7,13,0,0,675,676,7,12,0,0,676, + 677,7,8,0,0,677,678,7,1,0,0,678,130,1,0,0,0,679,680,7,4,0,0,680,681,7, + 7,0,0,681,132,1,0,0,0,682,683,7,12,0,0,683,684,7,14,0,0,684,685,7,14,0, + 0,685,134,1,0,0,0,686,687,7,14,0,0,687,688,7,6,0,0,688,689,7,7,0,0,689, + 690,7,11,0,0,690,136,1,0,0,0,691,692,7,3,0,0,692,693,7,7,0,0,693,694,7, + 2,0,0,694,695,7,22,0,0,695,696,7,8,0,0,696,697,7,13,0,0,697,138,1,0,0, + 0,698,699,7,18,0,0,699,700,7,13,0,0,700,701,7,0,0,0,701,702,7,1,0,0,702, + 703,7,6,0,0,703,704,7,4,0,0,704,140,1,0,0,0,705,706,7,18,0,0,706,707,7, + 13,0,0,707,708,7,4,0,0,708,709,7,7,0,0,709,142,1,0,0,0,710,711,7,17,0, + 0,711,712,7,12,0,0,712,713,7,2,0,0,713,714,7,22,0,0,714,715,7,1,0,0,715, + 716,7,0,0,0,716,144,1,0,0,0,717,718,7,11,0,0,718,719,7,6,0,0,719,720,7, + 18,0,0,720,721,7,8,0,0,721,722,7,12,0,0,722,723,7,6,0,0,723,724,7,24,0, + 0,724,146,1,0,0,0,725,726,7,20,0,0,726,727,7,1,0,0,727,728,7,24,0,0,728, + 148,1,0,0,0,729,730,7,3,0,0,730,731,7,7,0,0,731,732,7,13,0,0,732,733,7, + 0,0,0,733,734,7,4,0,0,734,735,7,6,0,0,735,736,7,12,0,0,736,737,7,18,0, + 0,737,738,7,13,0,0,738,739,7,4,0,0,739,150,1,0,0,0,740,741,7,5,0,0,741, + 742,7,7,0,0,742,743,7,6,0,0,743,744,7,1,0,0,744,745,7,18,0,0,745,746,7, + 21,0,0,746,747,7,13,0,0,747,152,1,0,0,0,748,749,7,6,0,0,749,750,7,1,0, + 0,750,751,7,5,0,0,751,752,7,1,0,0,752,753,7,6,0,0,753,754,7,1,0,0,754, + 755,7,13,0,0,755,756,7,3,0,0,756,757,7,1,0,0,757,758,7,0,0,0,758,154,1, + 0,0,0,759,760,7,14,0,0,760,761,7,1,0,0,761,762,7,2,0,0,762,763,7,1,0,0, + 763,764,7,4,0,0,764,765,7,1,0,0,765,156,1,0,0,0,766,767,7,22,0,0,767,768, + 7,11,0,0,768,769,7,14,0,0,769,770,7,12,0,0,770,771,7,4,0,0,771,772,7,1, + 0,0,772,158,1,0,0,0,773,774,7,3,0,0,774,775,7,12,0,0,775,776,7,0,0,0,776, + 777,7,3,0,0,777,778,7,12,0,0,778,779,7,14,0,0,779,780,7,1,0,0,780,160, + 1,0,0,0,781,782,7,6,0,0,782,783,7,1,0,0,783,784,7,0,0,0,784,785,7,4,0, + 0,785,786,7,6,0,0,786,787,7,18,0,0,787,788,7,3,0,0,788,789,7,4,0,0,789, + 162,1,0,0,0,790,791,7,12,0,0,791,792,7,3,0,0,792,793,7,4,0,0,793,794,7, + 18,0,0,794,795,7,7,0,0,795,796,7,13,0,0,796,164,1,0,0,0,797,798,7,0,0, + 0,798,799,7,1,0,0,799,800,7,4,0,0,800,166,1,0,0,0,801,802,7,14,0,0,802, + 803,7,1,0,0,803,804,7,5,0,0,804,805,7,12,0,0,805,806,7,22,0,0,806,807, + 7,2,0,0,807,808,7,4,0,0,808,168,1,0,0,0,809,810,7,13,0,0,810,811,7,7,0, + 0,811,170,1,0,0,0,812,813,7,22,0,0,813,814,7,13,0,0,814,815,7,18,0,0,815, + 816,7,16,0,0,816,817,7,22,0,0,817,818,7,1,0,0,818,172,1,0,0,0,819,820, + 7,3,0,0,820,821,7,2,0,0,821,822,7,22,0,0,822,823,7,0,0,0,823,824,7,4,0, + 0,824,825,7,1,0,0,825,826,7,6,0,0,826,827,7,1,0,0,827,828,7,14,0,0,828, + 174,1,0,0,0,829,830,7,18,0,0,830,831,7,14,0,0,831,832,7,1,0,0,832,833, + 7,13,0,0,833,834,7,4,0,0,834,835,7,18,0,0,835,836,7,4,0,0,836,837,7,24, + 0,0,837,176,1,0,0,0,838,839,7,13,0,0,839,840,7,7,0,0,840,841,7,13,0,0, + 841,842,7,3,0,0,842,843,7,2,0,0,843,844,7,22,0,0,844,845,7,0,0,0,845,846, + 7,4,0,0,846,847,7,1,0,0,847,848,7,6,0,0,848,849,7,1,0,0,849,850,7,14,0, + 0,850,178,1,0,0,0,851,852,7,18,0,0,852,853,7,13,0,0,853,854,7,14,0,0,854, + 855,7,1,0,0,855,856,7,15,0,0,856,180,1,0,0,0,857,858,7,4,0,0,858,859,7, + 1,0,0,859,860,7,8,0,0,860,861,7,11,0,0,861,862,7,7,0,0,862,863,7,6,0,0, + 863,864,7,12,0,0,864,865,7,6,0,0,865,866,7,24,0,0,866,182,1,0,0,0,867, + 868,7,9,0,0,868,869,7,18,0,0,869,870,7,4,0,0,870,871,7,10,0,0,871,184, + 1,0,0,0,872,873,7,3,0,0,873,874,7,7,0,0,874,875,7,8,0,0,875,876,7,11,0, + 0,876,877,7,6,0,0,877,878,7,1,0,0,878,879,7,0,0,0,879,880,7,0,0,0,880, + 881,7,18,0,0,881,882,7,7,0,0,882,883,7,13,0,0,883,186,1,0,0,0,884,885, + 7,3,0,0,885,886,7,7,0,0,886,887,7,8,0,0,887,888,7,11,0,0,888,188,1,0,0, + 0,889,890,7,14,0,0,890,891,7,18,0,0,891,892,7,0,0,0,892,893,7,12,0,0,893, + 894,7,2,0,0,894,895,7,2,0,0,895,896,7,7,0,0,896,897,7,9,0,0,897,190,1, + 0,0,0,898,899,7,18,0,0,899,900,7,21,0,0,900,901,7,13,0,0,901,902,7,7,0, + 0,902,903,7,6,0,0,903,904,7,1,0,0,904,192,1,0,0,0,905,906,7,3,0,0,906, + 907,7,10,0,0,907,908,7,1,0,0,908,909,7,3,0,0,909,910,7,20,0,0,910,194, + 1,0,0,0,911,912,7,17,0,0,912,913,7,18,0,0,913,914,7,1,0,0,914,915,7,9, + 0,0,915,196,1,0,0,0,916,917,7,11,0,0,917,918,7,6,0,0,918,919,7,7,0,0,919, + 920,7,3,0,0,920,921,7,1,0,0,921,922,7,14,0,0,922,923,7,22,0,0,923,924, + 7,6,0,0,924,925,7,1,0,0,925,198,1,0,0,0,926,927,7,11,0,0,927,928,7,12, + 0,0,928,929,7,6,0,0,929,930,7,12,0,0,930,931,7,8,0,0,931,932,7,1,0,0,932, + 933,7,4,0,0,933,934,7,1,0,0,934,935,7,6,0,0,935,936,7,0,0,0,936,200,1, + 0,0,0,937,938,7,1,0,0,938,939,7,15,0,0,939,940,7,1,0,0,940,941,7,3,0,0, + 941,942,7,22,0,0,942,943,7,4,0,0,943,944,7,1,0,0,944,202,1,0,0,0,945,946, + 7,1,0,0,946,947,7,15,0,0,947,948,7,1,0,0,948,949,7,3,0,0,949,204,1,0,0, + 0,950,951,7,12,0,0,951,952,7,0,0,0,952,953,7,3,0,0,953,206,1,0,0,0,954, + 955,7,14,0,0,955,956,7,1,0,0,956,957,7,0,0,0,957,958,7,3,0,0,958,208,1, + 0,0,0,959,960,7,4,0,0,960,961,7,6,0,0,961,962,7,22,0,0,962,963,7,1,0,0, + 963,210,1,0,0,0,964,965,7,5,0,0,965,966,7,12,0,0,966,967,7,2,0,0,967,968, + 7,0,0,0,968,969,7,1,0,0,969,212,1,0,0,0,970,971,7,13,0,0,971,972,7,22, + 0,0,972,973,7,2,0,0,973,974,7,2,0,0,974,214,1,0,0,0,975,976,7,6,0,0,976, + 977,7,12,0,0,977,978,7,13,0,0,978,979,7,21,0,0,979,980,7,1,0,0,980,216, + 1,0,0,0,981,982,7,21,0,0,982,983,7,6,0,0,983,984,7,7,0,0,984,985,7,22, + 0,0,985,986,7,11,0,0,986,987,7,0,0,0,987,218,1,0,0,0,988,989,7,22,0,0, + 989,990,7,13,0,0,990,991,7,19,0,0,991,992,7,7,0,0,992,993,7,22,0,0,993, + 994,7,13,0,0,994,995,7,14,0,0,995,996,7,1,0,0,996,997,7,14,0,0,997,220, + 1,0,0,0,998,999,7,11,0,0,999,1000,7,6,0,0,1000,1001,7,1,0,0,1001,1002, + 7,3,0,0,1002,1003,7,1,0,0,1003,1004,7,14,0,0,1004,1005,7,18,0,0,1005,1006, + 7,13,0,0,1006,1007,7,21,0,0,1007,222,1,0,0,0,1008,1009,7,5,0,0,1009,1010, + 7,7,0,0,1010,1011,7,2,0,0,1011,1012,7,2,0,0,1012,1013,7,7,0,0,1013,1014, + 7,9,0,0,1014,1015,7,18,0,0,1015,1016,7,13,0,0,1016,1017,7,21,0,0,1017, + 224,1,0,0,0,1018,1019,7,3,0,0,1019,1020,7,22,0,0,1020,1021,7,6,0,0,1021, + 1022,7,6,0,0,1022,1023,7,1,0,0,1023,1024,7,13,0,0,1024,1025,7,4,0,0,1025, + 226,1,0,0,0,1026,1027,7,1,0,0,1027,1028,7,15,0,0,1028,1029,7,3,0,0,1029, + 1030,7,2,0,0,1030,1031,7,22,0,0,1031,1032,7,14,0,0,1032,1033,7,1,0,0,1033, + 228,1,0,0,0,1034,1035,7,4,0,0,1035,1036,7,18,0,0,1036,1037,7,1,0,0,1037, + 1038,7,0,0,0,1038,230,1,0,0,0,1039,1040,7,7,0,0,1040,1041,7,4,0,0,1041, + 1042,7,10,0,0,1042,1043,7,1,0,0,1043,1044,7,6,0,0,1044,1045,7,0,0,0,1045, + 232,1,0,0,0,1046,1047,7,9,0,0,1047,1048,7,18,0,0,1048,1049,7,4,0,0,1049, + 1050,7,10,0,0,1050,1051,7,18,0,0,1051,1052,7,13,0,0,1052,234,1,0,0,0,1053, + 1054,7,2,0,0,1054,1055,7,12,0,0,1055,1056,7,0,0,0,1056,1057,7,4,0,0,1057, + 236,1,0,0,0,1058,1059,7,6,0,0,1059,1060,7,1,0,0,1060,1061,7,0,0,0,1061, + 1062,7,11,0,0,1062,1063,7,1,0,0,1063,1064,7,3,0,0,1064,1065,7,4,0,0,1065, + 238,1,0,0,0,1066,1067,7,13,0,0,1067,1068,7,22,0,0,1068,1069,7,2,0,0,1069, + 1070,7,2,0,0,1070,1071,7,0,0,0,1071,240,1,0,0,0,1072,1073,7,5,0,0,1073, + 1074,7,18,0,0,1074,1075,7,2,0,0,1075,1076,7,4,0,0,1076,1077,7,1,0,0,1077, + 1078,7,6,0,0,1078,242,1,0,0,0,1079,1080,5,42,0,0,1080,244,1,0,0,0,1081, + 1082,5,47,0,0,1082,246,1,0,0,0,1083,1084,5,92,0,0,1084,248,1,0,0,0,1085, + 1086,5,94,0,0,1086,250,1,0,0,0,1087,1088,5,43,0,0,1088,252,1,0,0,0,1089, + 1090,5,45,0,0,1090,254,1,0,0,0,1091,1092,5,38,0,0,1092,256,1,0,0,0,1093, + 1094,5,61,0,0,1094,258,1,0,0,0,1095,1096,5,60,0,0,1096,1100,5,62,0,0,1097, + 1098,5,33,0,0,1098,1100,5,61,0,0,1099,1095,1,0,0,0,1099,1097,1,0,0,0,1100, + 260,1,0,0,0,1101,1102,5,60,0,0,1102,1103,5,61,0,0,1103,262,1,0,0,0,1104, + 1105,5,62,0,0,1105,1106,5,61,0,0,1106,264,1,0,0,0,1107,1108,5,60,0,0,1108, + 266,1,0,0,0,1109,1110,5,62,0,0,1110,268,1,0,0,0,1111,1112,5,40,0,0,1112, + 270,1,0,0,0,1113,1114,5,41,0,0,1114,272,1,0,0,0,1115,1116,5,44,0,0,1116, + 274,1,0,0,0,1117,1118,5,46,0,0,1118,276,1,0,0,0,1119,1120,5,59,0,0,1120, + 278,1,0,0,0,1121,1122,5,64,0,0,1122,1123,5,64,0,0,1123,1124,1,0,0,0,1124, + 1128,7,25,0,0,1125,1127,7,26,0,0,1126,1125,1,0,0,0,1127,1130,1,0,0,0,1128, + 1126,1,0,0,0,1128,1129,1,0,0,0,1129,280,1,0,0,0,1130,1128,1,0,0,0,1131, + 1141,5,63,0,0,1132,1133,5,64,0,0,1133,1137,7,25,0,0,1134,1136,7,26,0,0, + 1135,1134,1,0,0,0,1136,1139,1,0,0,0,1137,1135,1,0,0,0,1137,1138,1,0,0, + 0,1138,1141,1,0,0,0,1139,1137,1,0,0,0,1140,1131,1,0,0,0,1140,1132,1,0, + 0,0,1141,282,1,0,0,0,1142,1143,5,48,0,0,1143,1145,7,15,0,0,1144,1146,7, + 27,0,0,1145,1144,1,0,0,0,1146,1147,1,0,0,0,1147,1145,1,0,0,0,1147,1148, + 1,0,0,0,1148,284,1,0,0,0,1149,1151,7,28,0,0,1150,1149,1,0,0,0,1151,1152, + 1,0,0,0,1152,1150,1,0,0,0,1152,1153,1,0,0,0,1153,286,1,0,0,0,1154,1156, + 7,28,0,0,1155,1154,1,0,0,0,1156,1157,1,0,0,0,1157,1155,1,0,0,0,1157,1158, + 1,0,0,0,1158,1159,1,0,0,0,1159,1163,5,46,0,0,1160,1162,7,28,0,0,1161,1160, + 1,0,0,0,1162,1165,1,0,0,0,1163,1161,1,0,0,0,1163,1164,1,0,0,0,1164,1167, + 1,0,0,0,1165,1163,1,0,0,0,1166,1168,3,289,144,0,1167,1166,1,0,0,0,1167, + 1168,1,0,0,0,1168,1185,1,0,0,0,1169,1171,5,46,0,0,1170,1172,7,28,0,0,1171, + 1170,1,0,0,0,1172,1173,1,0,0,0,1173,1171,1,0,0,0,1173,1174,1,0,0,0,1174, + 1176,1,0,0,0,1175,1177,3,289,144,0,1176,1175,1,0,0,0,1176,1177,1,0,0,0, + 1177,1185,1,0,0,0,1178,1180,7,28,0,0,1179,1178,1,0,0,0,1180,1181,1,0,0, + 0,1181,1179,1,0,0,0,1181,1182,1,0,0,0,1182,1183,1,0,0,0,1183,1185,3,289, + 144,0,1184,1155,1,0,0,0,1184,1169,1,0,0,0,1184,1179,1,0,0,0,1185,288,1, + 0,0,0,1186,1188,7,1,0,0,1187,1189,7,29,0,0,1188,1187,1,0,0,0,1188,1189, + 1,0,0,0,1189,1191,1,0,0,0,1190,1192,7,28,0,0,1191,1190,1,0,0,0,1192,1193, + 1,0,0,0,1193,1191,1,0,0,0,1193,1194,1,0,0,0,1194,290,1,0,0,0,1195,1201, + 5,34,0,0,1196,1200,8,30,0,0,1197,1198,5,34,0,0,1198,1200,5,34,0,0,1199, + 1196,1,0,0,0,1199,1197,1,0,0,0,1200,1203,1,0,0,0,1201,1199,1,0,0,0,1201, + 1202,1,0,0,0,1202,1204,1,0,0,0,1203,1201,1,0,0,0,1204,1216,5,34,0,0,1205, + 1211,5,39,0,0,1206,1210,8,31,0,0,1207,1208,5,39,0,0,1208,1210,5,39,0,0, + 1209,1206,1,0,0,0,1209,1207,1,0,0,0,1210,1213,1,0,0,0,1211,1209,1,0,0, + 0,1211,1212,1,0,0,0,1212,1214,1,0,0,0,1213,1211,1,0,0,0,1214,1216,5,39, + 0,0,1215,1195,1,0,0,0,1215,1205,1,0,0,0,1216,292,1,0,0,0,1217,1221,5,35, + 0,0,1218,1220,8,32,0,0,1219,1218,1,0,0,0,1220,1223,1,0,0,0,1221,1219,1, + 0,0,0,1221,1222,1,0,0,0,1222,1224,1,0,0,0,1223,1221,1,0,0,0,1224,1225, + 5,35,0,0,1225,294,1,0,0,0,1226,1228,5,123,0,0,1227,1229,3,297,148,0,1228, + 1227,1,0,0,0,1229,1230,1,0,0,0,1230,1228,1,0,0,0,1230,1231,1,0,0,0,1231, + 1232,1,0,0,0,1232,1234,5,45,0,0,1233,1235,3,297,148,0,1234,1233,1,0,0, + 0,1235,1236,1,0,0,0,1236,1234,1,0,0,0,1236,1237,1,0,0,0,1237,1238,1,0, + 0,0,1238,1240,5,45,0,0,1239,1241,3,297,148,0,1240,1239,1,0,0,0,1241,1242, + 1,0,0,0,1242,1240,1,0,0,0,1242,1243,1,0,0,0,1243,1244,1,0,0,0,1244,1246, + 5,45,0,0,1245,1247,3,297,148,0,1246,1245,1,0,0,0,1247,1248,1,0,0,0,1248, + 1246,1,0,0,0,1248,1249,1,0,0,0,1249,1250,1,0,0,0,1250,1252,5,45,0,0,1251, + 1253,3,297,148,0,1252,1251,1,0,0,0,1253,1254,1,0,0,0,1254,1252,1,0,0,0, + 1254,1255,1,0,0,0,1255,1256,1,0,0,0,1256,1257,5,125,0,0,1257,296,1,0,0, + 0,1258,1259,7,27,0,0,1259,298,1,0,0,0,1260,1262,5,91,0,0,1261,1263,8,33, + 0,0,1262,1261,1,0,0,0,1263,1264,1,0,0,0,1264,1262,1,0,0,0,1264,1265,1, + 0,0,0,1265,1266,1,0,0,0,1266,1267,5,93,0,0,1267,300,1,0,0,0,1268,1270, + 5,96,0,0,1269,1271,8,34,0,0,1270,1269,1,0,0,0,1271,1272,1,0,0,0,1272,1270, + 1,0,0,0,1272,1273,1,0,0,0,1273,1274,1,0,0,0,1274,1275,5,96,0,0,1275,302, + 1,0,0,0,1276,1280,7,25,0,0,1277,1279,7,26,0,0,1278,1277,1,0,0,0,1279,1282, + 1,0,0,0,1280,1278,1,0,0,0,1280,1281,1,0,0,0,1281,1284,1,0,0,0,1282,1280, + 1,0,0,0,1283,1285,5,36,0,0,1284,1283,1,0,0,0,1284,1285,1,0,0,0,1285,304, + 1,0,0,0,1286,1288,7,35,0,0,1287,1286,1,0,0,0,1288,1289,1,0,0,0,1289,1287, + 1,0,0,0,1289,1290,1,0,0,0,1290,1291,1,0,0,0,1291,1292,6,152,0,0,1292,306, + 1,0,0,0,1293,1294,5,45,0,0,1294,1295,5,45,0,0,1295,1296,1,0,0,0,1296,1300, + 4,153,0,0,1297,1299,8,36,0,0,1298,1297,1,0,0,0,1299,1302,1,0,0,0,1300, + 1298,1,0,0,0,1300,1301,1,0,0,0,1301,1303,1,0,0,0,1302,1300,1,0,0,0,1303, + 1304,6,153,0,0,1304,308,1,0,0,0,1305,1306,5,47,0,0,1306,1307,5,42,0,0, + 1307,1311,1,0,0,0,1308,1310,9,0,0,0,1309,1308,1,0,0,0,1310,1313,1,0,0, + 0,1311,1312,1,0,0,0,1311,1309,1,0,0,0,1312,1314,1,0,0,0,1313,1311,1,0, + 0,0,1314,1315,5,42,0,0,1315,1316,5,47,0,0,1316,1317,1,0,0,0,1317,1318, + 6,154,0,0,1318,310,1,0,0,0,34,0,1099,1128,1137,1140,1147,1152,1157,1163, + 1167,1173,1176,1181,1184,1188,1193,1199,1201,1209,1211,1215,1221,1230, + 1236,1242,1248,1254,1264,1272,1280,1284,1289,1300,1311,1,6,0,0 }; public static readonly ATN _ATN = diff --git a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlParser.cs b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlParser.cs index 51833008e..d70f31e88 100644 --- a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlParser.cs +++ b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlParser.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -// Generated from AccessSql.g4 by ANTLR 4.13.1 +// Generated from D:/toolkits/efcorejetlibred/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 by ANTLR 4.13.1 // Unreachable code detected #pragma warning disable 0162 @@ -37,25 +37,29 @@ public partial class AccessSqlParser : Parser { protected static DFA[] decisionToDFA; protected static PredictionContextCache sharedContextCache = new PredictionContextCache(); public const int - SELECT=1, FROM=2, WHERE=3, TOP=4, AS=5, AND=6, OR=7, NOT=8, BAND=9, BOR=10, - BXOR=11, BNOT=12, LIKE=13, MOD=14, INNER=15, LEFT=16, RIGHT=17, FULL=18, - OUTER=19, JOIN=20, IN=21, ON=22, ORDER=23, GROUP=24, IS=25, BY=26, HAVING=27, - EXISTS=28, IF=29, THEN=30, DISTINCTROW=31, DISTINCT=32, PERCENT=33, CROSS=34, - APPLY=35, OVER=36, PARTITION=37, CASE=38, WHEN=39, ELSE=40, END=41, OFFSET=42, - FETCH=43, NEXT=44, FIRST=45, ROWS=46, ROW=47, ONLY=48, BETWEEN=49, UNION=50, - ALL=51, INTERSECT=52, EXCEPT=53, CREATE=54, TABLE=55, BEGIN=56, COMMIT=57, - ROLLBACK=58, TRANSACTION=59, WORK=60, ALTER=61, RENAME=62, TO=63, ADD=64, - DROP=65, COLUMN=66, INSERT=67, INTO=68, VALUES=69, PRIMARY=70, KEY=71, - CONSTRAINT=72, FOREIGN=73, REFERENCES=74, DELETE=75, UPDATE=76, CASCADE=77, - RESTRICT=78, ACTION=79, SET=80, DEFAULT=81, NO=82, UNIQUE=83, INDEX=84, - TEMPORARY=85, WITH=86, COMPRESSION=87, COMP=88, DISALLOW=89, IGNORE=90, - CHECK=91, VIEW=92, PROCEDURE=93, PARAMETERS=94, EXECUTE=95, EXEC=96, ASC=97, - DESC=98, TRUE=99, FALSE=100, NULL=101, STAR=102, SLASH=103, BACKSLASH=104, - CARET=105, PLUS=106, MINUS=107, AMP=108, EQ=109, NEQ=110, LTE=111, GTE=112, - LT=113, GT=114, LPAREN=115, RPAREN=116, COMMA=117, DOT=118, SEMI=119, - SYSVAR=120, PARAM=121, HEX_LITERAL=122, INTEGER_LITERAL=123, NUMBER_LITERAL=124, - STRING_LITERAL=125, DATE_LITERAL=126, GUID_LITERAL=127, BRACKET_ID=128, - BACKTICK_ID=129, IDENTIFIER=130, WS=131, LINE_COMMENT=132, BLOCK_COMMENT=133; + SELECT=1, FROM=2, WHERE=3, TOP=4, AS=5, AND=6, OR=7, NOT=8, XOR=9, EQV=10, + IMP=11, BAND=12, BOR=13, BXOR=14, BNOT=15, LIKE=16, MOD=17, INNER=18, + LEFT=19, RIGHT=20, FULL=21, OUTER=22, JOIN=23, IN=24, ON=25, ORDER=26, + GROUP=27, IS=28, BY=29, HAVING=30, EXISTS=31, IF=32, THEN=33, DISTINCTROW=34, + DISTINCT=35, PERCENT=36, CROSS=37, APPLY=38, OVER=39, PARTITION=40, CASE=41, + WHEN=42, ELSE=43, END=44, OFFSET=45, FETCH=46, NEXT=47, FIRST=48, ROWS=49, + ROW=50, ONLY=51, BETWEEN=52, UNION=53, ALL=54, INTERSECT=55, EXCEPT=56, + CREATE=57, TABLE=58, BEGIN=59, COMMIT=60, ROLLBACK=61, TRANSACTION=62, + WORK=63, ALTER=64, RENAME=65, TO=66, ADD=67, DROP=68, COLUMN=69, INSERT=70, + INTO=71, VALUES=72, PRIMARY=73, KEY=74, CONSTRAINT=75, FOREIGN=76, REFERENCES=77, + DELETE=78, UPDATE=79, CASCADE=80, RESTRICT=81, ACTION=82, SET=83, DEFAULT=84, + NO=85, UNIQUE=86, CLUSTERED=87, IDENTITY=88, NONCLUSTERED=89, INDEX=90, + TEMPORARY=91, WITH=92, COMPRESSION=93, COMP=94, DISALLOW=95, IGNORE=96, + CHECK=97, VIEW=98, PROCEDURE=99, PARAMETERS=100, EXECUTE=101, EXEC=102, + ASC=103, DESC=104, TRUE=105, FALSE=106, NULL=107, RANGE=108, GROUPS=109, + UNBOUNDED=110, PRECEDING=111, FOLLOWING=112, CURRENT=113, EXCLUDE=114, + TIES=115, OTHERS=116, WITHIN=117, LAST=118, RESPECT=119, NULLS=120, FILTER=121, + STAR=122, SLASH=123, BACKSLASH=124, CARET=125, PLUS=126, MINUS=127, AMP=128, + EQ=129, NEQ=130, LTE=131, GTE=132, LT=133, GT=134, LPAREN=135, RPAREN=136, + COMMA=137, DOT=138, SEMI=139, SYSVAR=140, PARAM=141, HEX_LITERAL=142, + INTEGER_LITERAL=143, NUMBER_LITERAL=144, STRING_LITERAL=145, DATE_LITERAL=146, + GUID_LITERAL=147, BRACKET_ID=148, BACKTICK_ID=149, IDENTIFIER=150, WS=151, + LINE_COMMENT=152, BLOCK_COMMENT=153; public const int RULE_statement = 0, RULE_ifThenStatement = 1, RULE_thenBody = 2, RULE_executeStatement = 3, RULE_updateStatement = 4, RULE_assignment = 5, RULE_deleteStatement = 6, @@ -77,7 +81,9 @@ public const int RULE_expression = 56, RULE_primary = 57, RULE_caseExpression = 58, RULE_caseWhen = 59, RULE_functionCall = 60, RULE_functionName = 61, RULE_columnRef = 62, RULE_identifier = 63, RULE_literal = 64, RULE_transactionStatement = 65, RULE_standaloneExpression = 66, - RULE_windowSpecification = 67; + RULE_windowSpecification = 67, RULE_clusteredOption = 68, RULE_windowFrame = 69, + RULE_frameBound = 70, RULE_frameExclusion = 71, RULE_nonReservedKeyword = 72, + RULE_filterClause = 73, RULE_withinGroup = 74, RULE_nthRowFrom = 75, RULE_nullTreatment = 76; public static readonly string[] ruleNames = { "statement", "ifThenStatement", "thenBody", "executeStatement", "updateStatement", "assignment", "deleteStatement", "systemVariableSelect", "sysVarItem", @@ -93,7 +99,9 @@ public const int "joinClause", "joinType", "whereClause", "orderByClause", "orderByItem", "expression", "primary", "caseExpression", "caseWhen", "functionCall", "functionName", "columnRef", "identifier", "literal", "transactionStatement", - "standaloneExpression", "windowSpecification" + "standaloneExpression", "windowSpecification", "clusteredOption", "windowFrame", + "frameBound", "frameExclusion", "nonReservedKeyword", "filterClause", + "withinGroup", "nthRowFrom", "nullTreatment" }; private static readonly string[] _LiteralNames = { @@ -105,26 +113,30 @@ public const int null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, "'*'", "'/'", "'\\'", "'^'", "'+'", - "'-'", "'&'", "'='", null, "'<='", "'>='", "'<'", "'>'", "'('", "')'", - "','", "'.'", "';'" + null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, + null, null, "'*'", "'/'", "'\\'", "'^'", "'+'", "'-'", "'&'", "'='", null, + "'<='", "'>='", "'<'", "'>'", "'('", "')'", "','", "'.'", "';'" }; private static readonly string[] _SymbolicNames = { - null, "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "BAND", - "BOR", "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", "RIGHT", "FULL", - "OUTER", "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", "BY", "HAVING", "EXISTS", - "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", "CROSS", "APPLY", - "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", "OFFSET", "FETCH", - "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", "UNION", "ALL", "INTERSECT", - "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", "ROLLBACK", "TRANSACTION", - "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", "COLUMN", "INSERT", "INTO", - "VALUES", "PRIMARY", "KEY", "CONSTRAINT", "FOREIGN", "REFERENCES", "DELETE", - "UPDATE", "CASCADE", "RESTRICT", "ACTION", "SET", "DEFAULT", "NO", "UNIQUE", + null, "SELECT", "FROM", "WHERE", "TOP", "AS", "AND", "OR", "NOT", "XOR", + "EQV", "IMP", "BAND", "BOR", "BXOR", "BNOT", "LIKE", "MOD", "INNER", "LEFT", + "RIGHT", "FULL", "OUTER", "JOIN", "IN", "ON", "ORDER", "GROUP", "IS", + "BY", "HAVING", "EXISTS", "IF", "THEN", "DISTINCTROW", "DISTINCT", "PERCENT", + "CROSS", "APPLY", "OVER", "PARTITION", "CASE", "WHEN", "ELSE", "END", + "OFFSET", "FETCH", "NEXT", "FIRST", "ROWS", "ROW", "ONLY", "BETWEEN", + "UNION", "ALL", "INTERSECT", "EXCEPT", "CREATE", "TABLE", "BEGIN", "COMMIT", + "ROLLBACK", "TRANSACTION", "WORK", "ALTER", "RENAME", "TO", "ADD", "DROP", + "COLUMN", "INSERT", "INTO", "VALUES", "PRIMARY", "KEY", "CONSTRAINT", + "FOREIGN", "REFERENCES", "DELETE", "UPDATE", "CASCADE", "RESTRICT", "ACTION", + "SET", "DEFAULT", "NO", "UNIQUE", "CLUSTERED", "IDENTITY", "NONCLUSTERED", "INDEX", "TEMPORARY", "WITH", "COMPRESSION", "COMP", "DISALLOW", "IGNORE", "CHECK", "VIEW", "PROCEDURE", "PARAMETERS", "EXECUTE", "EXEC", "ASC", - "DESC", "TRUE", "FALSE", "NULL", "STAR", "SLASH", "BACKSLASH", "CARET", - "PLUS", "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", - "RPAREN", "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", + "DESC", "TRUE", "FALSE", "NULL", "RANGE", "GROUPS", "UNBOUNDED", "PRECEDING", + "FOLLOWING", "CURRENT", "EXCLUDE", "TIES", "OTHERS", "WITHIN", "LAST", + "RESPECT", "NULLS", "FILTER", "STAR", "SLASH", "BACKSLASH", "CARET", "PLUS", + "MINUS", "AMP", "EQ", "NEQ", "LTE", "GTE", "LT", "GT", "LPAREN", "RPAREN", + "COMMA", "DOT", "SEMI", "SYSVAR", "PARAM", "HEX_LITERAL", "INTEGER_LITERAL", "NUMBER_LITERAL", "STRING_LITERAL", "DATE_LITERAL", "GUID_LITERAL", "BRACKET_ID", "BACKTICK_ID", "IDENTIFIER", "WS", "LINE_COMMENT", "BLOCK_COMMENT" }; @@ -152,6 +164,18 @@ static AccessSqlParser() { } } + + // The precedence the generated parser gives a BETWEEN's upper bound: that of the comparisons, one above BETWEEN. + private const int BetweenBoundPrecedence = 13; + + // ANTLR parses the middle operand of an alternative at precedence 0 and has no syntax to raise it, which lets a + // BETWEEN's lower bound swallow the AND that ends it: `x BETWEEN 1 AND 10 AND y = 2` would read as + // `x BETWEEN (1 AND 10) AND (y = 2)`. The lower bound is the only operand parsed while its BETWEEN has no lower + // bound yet, so it is given the upper bound's precedence here. + public override void EnterRecursionRule(ParserRuleContext localctx, int state, int ruleIndex, int precedence) => + base.EnterRecursionRule(localctx, state, ruleIndex, + localctx.Parent is BetweenExprContext { lo: null } ? Math.Max(precedence, BetweenBoundPrecedence) : precedence); + public AccessSqlParser(ITokenStream input) : this(input, Console.Out, Console.Error) { } public AccessSqlParser(ITokenStream input, TextWriter output, TextWriter errorOutput) @@ -229,115 +253,115 @@ public StatementContext statement() { try { EnterOuterAlt(_localctx, 1); { - State = 137; + State = 155; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==PARAMETERS) { { - State = 136; + State = 154; parametersClause(); } } - State = 153; + State = 171; ErrorHandler.Sync(this); switch ( Interpreter.AdaptivePredict(TokenStream,1,Context) ) { case 1: { - State = 139; + State = 157; ifThenStatement(); } break; case 2: { - State = 140; + State = 158; createTableStatement(); } break; case 3: { - State = 141; + State = 159; createIndexStatement(); } break; case 4: { - State = 142; + State = 160; createViewStatement(); } break; case 5: { - State = 143; + State = 161; createProcedureStatement(); } break; case 6: { - State = 144; + State = 162; alterTableStatement(); } break; case 7: { - State = 145; + State = 163; dropStatement(); } break; case 8: { - State = 146; + State = 164; insertStatement(); } break; case 9: { - State = 147; + State = 165; updateStatement(); } break; case 10: { - State = 148; + State = 166; deleteStatement(); } break; case 11: { - State = 149; + State = 167; transactionStatement(); } break; case 12: { - State = 150; + State = 168; executeStatement(); } break; case 13: { - State = 151; + State = 169; systemVariableSelect(); } break; case 14: { - State = 152; + State = 170; queryExpression(); } break; } - State = 156; + State = 174; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==SEMI) { { - State = 155; + State = 173; Match(SEMI); } } - State = 158; + State = 176; Match(Eof); } } @@ -387,29 +411,29 @@ public IfThenStatementContext ifThenStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 160; + State = 178; Match(IF); - State = 162; + State = 180; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 161; + State = 179; _localctx.not = Match(NOT); } } - State = 164; + State = 182; Match(EXISTS); - State = 165; + State = 183; Match(LPAREN); - State = 166; + State = 184; queryExpression(); - State = 167; + State = 185; Match(RPAREN); - State = 168; + State = 186; Match(THEN); - State = 169; + State = 187; thenBody(); } } @@ -476,83 +500,83 @@ public ThenBodyContext thenBody() { ThenBodyContext _localctx = new ThenBodyContext(Context, State); EnterRule(_localctx, 4, RULE_thenBody); try { - State = 182; + State = 200; ErrorHandler.Sync(this); switch ( Interpreter.AdaptivePredict(TokenStream,4,Context) ) { case 1: EnterOuterAlt(_localctx, 1); { - State = 171; + State = 189; createTableStatement(); } break; case 2: EnterOuterAlt(_localctx, 2); { - State = 172; + State = 190; createIndexStatement(); } break; case 3: EnterOuterAlt(_localctx, 3); { - State = 173; + State = 191; createViewStatement(); } break; case 4: EnterOuterAlt(_localctx, 4); { - State = 174; + State = 192; createProcedureStatement(); } break; case 5: EnterOuterAlt(_localctx, 5); { - State = 175; + State = 193; alterTableStatement(); } break; case 6: EnterOuterAlt(_localctx, 6); { - State = 176; + State = 194; dropStatement(); } break; case 7: EnterOuterAlt(_localctx, 7); { - State = 177; + State = 195; insertStatement(); } break; case 8: EnterOuterAlt(_localctx, 8); { - State = 178; + State = 196; updateStatement(); } break; case 9: EnterOuterAlt(_localctx, 9); { - State = 179; + State = 197; deleteStatement(); } break; case 10: EnterOuterAlt(_localctx, 10); { - State = 180; + State = 198; executeStatement(); } break; case 11: EnterOuterAlt(_localctx, 11); { - State = 181; + State = 199; queryExpression(); } break; @@ -607,7 +631,7 @@ public ExecuteStatementContext executeStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 184; + State = 202; _la = TokenStream.LA(1); if ( !(_la==EXECUTE || _la==EXEC) ) { ErrorHandler.RecoverInline(this); @@ -616,28 +640,28 @@ public ExecuteStatementContext executeStatement() { ErrorHandler.ReportMatch(this); Consume(); } - State = 185; + State = 203; _localctx.name = identifier(); - State = 194; + State = 212; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 35596957585664L) != 0) || ((((_la - 97)) & ~0x3f) == 0 && ((1L << (_la - 97)) & 17171743773L) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 284775660683520L) != 0) || ((((_la - 103)) & ~0x3f) == 0 && ((1L << (_la - 103)) & 281341858414589L) != 0)) { { - State = 186; + State = 204; expression(0); - State = 191; + State = 209; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 187; + State = 205; Match(COMMA); - State = 188; + State = 206; expression(0); } } - State = 193; + State = 211; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -659,8 +683,11 @@ public ExecuteStatementContext executeStatement() { public partial class UpdateStatementContext : ParserRuleContext { [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode UPDATE() { return GetToken(AccessSqlParser.UPDATE, 0); } - [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext tableSource() { - return GetRuleContext(0); + [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext[] tableSource() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext tableSource(int i) { + return GetRuleContext(i); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode SET() { return GetToken(AccessSqlParser.SET, 0); } [System.Diagnostics.DebuggerNonUserCode] public AssignmentContext[] assignment() { @@ -697,36 +724,52 @@ public UpdateStatementContext updateStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 196; + State = 214; Match(UPDATE); - State = 197; + State = 215; tableSource(); - State = 198; + State = 220; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + while (_la==COMMA) { + { + { + State = 216; + Match(COMMA); + State = 217; + tableSource(); + } + } + State = 222; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + } + State = 223; Match(SET); - State = 199; + State = 224; assignment(); - State = 204; + State = 229; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 200; + State = 225; Match(COMMA); - State = 201; + State = 226; assignment(); } } - State = 206; + State = 231; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 208; + State = 233; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==WHERE) { { - State = 207; + State = 232; whereClause(); } } @@ -773,11 +816,11 @@ public AssignmentContext assignment() { try { EnterOuterAlt(_localctx, 1); { - State = 210; + State = 235; _localctx.target = columnRef(); - State = 211; + State = 236; Match(EQ); - State = 212; + State = 237; expression(0); } } @@ -796,11 +839,18 @@ public partial class DeleteStatementContext : ParserRuleContext { public IdentifierContext target; [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode DELETE() { return GetToken(AccessSqlParser.DELETE, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FROM() { return GetToken(AccessSqlParser.FROM, 0); } - [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext tableSource() { - return GetRuleContext(0); + [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext[] tableSource() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public TableSourceContext tableSource(int i) { + return GetRuleContext(i); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode DOT() { return GetToken(AccessSqlParser.DOT, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode STAR() { return GetToken(AccessSqlParser.STAR, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode[] COMMA() { return GetTokens(AccessSqlParser.COMMA); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode COMMA(int i) { + return GetToken(AccessSqlParser.COMMA, i); + } [System.Diagnostics.DebuggerNonUserCode] public WhereClauseContext whereClause() { return GetRuleContext(0); } @@ -828,26 +878,40 @@ public DeleteStatementContext deleteStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 214; + State = 239; Match(DELETE); - State = 220; + State = 245; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case BRACKET_ID: case BACKTICK_ID: case IDENTIFIER: { - State = 215; + State = 240; _localctx.target = identifier(); - State = 216; + State = 241; Match(DOT); - State = 217; + State = 242; Match(STAR); } break; case STAR: { - State = 219; + State = 244; Match(STAR); } break; @@ -856,16 +920,32 @@ public DeleteStatementContext deleteStatement() { default: break; } - State = 222; + State = 247; Match(FROM); - State = 223; + State = 248; tableSource(); - State = 225; + State = 253; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + while (_la==COMMA) { + { + { + State = 249; + Match(COMMA); + State = 250; + tableSource(); + } + } + State = 255; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + } + State = 257; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==WHERE) { { - State = 224; + State = 256; whereClause(); } } @@ -916,23 +996,23 @@ public SystemVariableSelectContext systemVariableSelect() { try { EnterOuterAlt(_localctx, 1); { - State = 227; + State = 259; Match(SELECT); - State = 228; + State = 260; sysVarItem(); - State = 233; + State = 265; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 229; + State = 261; Match(COMMA); - State = 230; + State = 262; sysVarItem(); } } - State = 235; + State = 267; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -977,24 +1057,24 @@ public SysVarItemContext sysVarItem() { try { EnterOuterAlt(_localctx, 1); { - State = 236; + State = 268; Match(SYSVAR); - State = 241; + State = 273; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (_la==AS || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) { + if (_la==AS || ((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7696581410815L) != 0)) { { - State = 238; + State = 270; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==AS) { { - State = 237; + State = 269; Match(AS); } } - State = 240; + State = 272; _localctx.alias = identifier(); } } @@ -1046,27 +1126,27 @@ public ParametersClauseContext parametersClause() { try { EnterOuterAlt(_localctx, 1); { - State = 243; + State = 275; Match(PARAMETERS); - State = 244; + State = 276; procParam(); - State = 249; + State = 281; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 245; + State = 277; Match(COMMA); - State = 246; + State = 278; procParam(); } } - State = 251; + State = 283; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 252; + State = 284; Match(SEMI); } } @@ -1130,61 +1210,61 @@ public CreateTableStatementContext createTableStatement() { int _alt; EnterOuterAlt(_localctx, 1); { - State = 254; + State = 286; Match(CREATE); - State = 256; + State = 288; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==TEMPORARY) { { - State = 255; + State = 287; _localctx.temp = Match(TEMPORARY); } } - State = 258; + State = 290; Match(TABLE); - State = 259; + State = 291; _localctx.table = identifier(); - State = 260; + State = 292; Match(LPAREN); - State = 261; + State = 293; columnDefinition(); - State = 266; + State = 298; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,16,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,18,Context); while ( _alt!=2 && _alt!=global::Antlr4.Runtime.Atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - State = 262; + State = 294; Match(COMMA); - State = 263; + State = 295; columnDefinition(); } } } - State = 268; + State = 300; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,16,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,18,Context); } - State = 273; + State = 305; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 269; + State = 301; Match(COMMA); - State = 270; + State = 302; tableConstraint(); } } - State = 275; + State = 307; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 276; + State = 308; Match(RPAREN); } } @@ -1243,47 +1323,47 @@ public CreateViewStatementContext createViewStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 278; + State = 310; Match(CREATE); - State = 279; + State = 311; Match(VIEW); - State = 280; + State = 312; _localctx.name = identifier(); - State = 292; + State = 324; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==LPAREN) { { - State = 281; + State = 313; Match(LPAREN); - State = 282; + State = 314; _localctx._identifier = identifier(); _localctx._columns.Add(_localctx._identifier); - State = 287; + State = 319; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 283; + State = 315; Match(COMMA); - State = 284; + State = 316; _localctx._identifier = identifier(); _localctx._columns.Add(_localctx._identifier); } } - State = 289; + State = 321; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 290; + State = 322; Match(RPAREN); } } - State = 294; + State = 326; Match(AS); - State = 295; + State = 327; _localctx.query = queryExpression(); } } @@ -1334,25 +1414,25 @@ public CreateProcedureStatementContext createProcedureStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 297; + State = 329; Match(CREATE); - State = 298; + State = 330; Match(PROCEDURE); - State = 299; + State = 331; _localctx.name = identifier(); - State = 301; + State = 333; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (((((_la - 115)) & ~0x3f) == 0 && ((1L << (_la - 115)) & 57409L) != 0)) { + if (((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7705305563135L) != 0)) { { - State = 300; + State = 332; procParamList(); } } - State = 303; + State = 335; Match(AS); - State = 304; + State = 336; _localctx.body = procedureBody(); } } @@ -1399,57 +1479,71 @@ public ProcParamListContext procParamList() { EnterRule(_localctx, 26, RULE_procParamList); int _la; try { - State = 325; + State = 357; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case LPAREN: EnterOuterAlt(_localctx, 1); { - State = 306; + State = 338; Match(LPAREN); - State = 307; + State = 339; procParam(); - State = 312; + State = 344; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 308; + State = 340; Match(COMMA); - State = 309; + State = 341; procParam(); } } - State = 314; + State = 346; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 315; + State = 347; Match(RPAREN); } break; + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case PARAM: case BRACKET_ID: case BACKTICK_ID: case IDENTIFIER: EnterOuterAlt(_localctx, 2); { - State = 317; + State = 349; procParam(); - State = 322; + State = 354; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 318; + State = 350; Match(COMMA); - State = 319; + State = 351; procParam(); } } - State = 324; + State = 356; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -1498,9 +1592,9 @@ public ProcParamContext procParam() { try { EnterOuterAlt(_localctx, 1); { - State = 327; + State = 359; _localctx.pname = procParamName(); - State = 328; + State = 360; dataType(); } } @@ -1538,22 +1632,36 @@ public ProcParamNameContext procParamName() { ProcParamNameContext _localctx = new ProcParamNameContext(Context, State); EnterRule(_localctx, 30, RULE_procParamName); try { - State = 332; + State = 364; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case BRACKET_ID: case BACKTICK_ID: case IDENTIFIER: EnterOuterAlt(_localctx, 1); { - State = 330; + State = 362; identifier(); } break; case PARAM: EnterOuterAlt(_localctx, 2); { - State = 331; + State = 363; Match(PARAM); } break; @@ -1602,13 +1710,13 @@ public AlterTableStatementContext alterTableStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 334; + State = 366; Match(ALTER); - State = 335; + State = 367; Match(TABLE); - State = 336; + State = 368; _localctx.table = identifier(); - State = 337; + State = 369; alterTableAction(); } } @@ -1815,26 +1923,26 @@ public AlterTableActionContext alterTableAction() { EnterRule(_localctx, 34, RULE_alterTableAction); int _la; try { - State = 396; + State = 428; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,30,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,32,Context) ) { case 1: _localctx = new AddColumnActionContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 339; + State = 371; Match(ADD); - State = 341; + State = 373; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==COLUMN) { { - State = 340; + State = 372; Match(COLUMN); } } - State = 343; + State = 375; columnDefinition(); } break; @@ -1842,9 +1950,9 @@ public AlterTableActionContext alterTableAction() { _localctx = new AddConstraintActionContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 344; + State = 376; Match(ADD); - State = 345; + State = 377; tableConstraint(); } break; @@ -1852,33 +1960,33 @@ public AlterTableActionContext alterTableAction() { _localctx = new AlterColumnActionContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 346; + State = 378; Match(ALTER); - State = 348; + State = 380; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==COLUMN) { { - State = 347; + State = 379; Match(COLUMN); } } - State = 350; + State = 382; ((AlterColumnActionContext)_localctx).field = identifier(); - State = 351; + State = 383; dataType(); - State = 355; + State = 387; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - while (_la==NOT || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & 2149656597L) != 0)) { + while (_la==NOT || ((((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 17197213717L) != 0)) { { { - State = 352; + State = 384; columnConstraint(); } } - State = 357; + State = 389; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -1888,25 +1996,25 @@ public AlterTableActionContext alterTableAction() { _localctx = new AlterColumnSetDefaultActionContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 358; + State = 390; Match(ALTER); - State = 360; + State = 392; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==COLUMN) { { - State = 359; + State = 391; Match(COLUMN); } } - State = 362; + State = 394; ((AlterColumnSetDefaultActionContext)_localctx).field = identifier(); - State = 363; + State = 395; Match(SET); - State = 364; + State = 396; Match(DEFAULT); - State = 365; + State = 397; expression(0); } break; @@ -1914,23 +2022,23 @@ public AlterTableActionContext alterTableAction() { _localctx = new AlterColumnDropDefaultActionContext(_localctx); EnterOuterAlt(_localctx, 5); { - State = 367; + State = 399; Match(ALTER); - State = 369; + State = 401; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==COLUMN) { { - State = 368; + State = 400; Match(COLUMN); } } - State = 371; + State = 403; ((AlterColumnDropDefaultActionContext)_localctx).field = identifier(); - State = 372; + State = 404; Match(DROP); - State = 373; + State = 405; Match(DEFAULT); } break; @@ -1938,11 +2046,11 @@ public AlterTableActionContext alterTableAction() { _localctx = new DropColumnActionContext(_localctx); EnterOuterAlt(_localctx, 6); { - State = 375; + State = 407; Match(DROP); - State = 376; + State = 408; Match(COLUMN); - State = 377; + State = 409; ((DropColumnActionContext)_localctx).field = identifier(); } break; @@ -1950,11 +2058,11 @@ public AlterTableActionContext alterTableAction() { _localctx = new DropConstraintActionContext(_localctx); EnterOuterAlt(_localctx, 7); { - State = 378; + State = 410; Match(DROP); - State = 379; + State = 411; Match(CONSTRAINT); - State = 380; + State = 412; ((DropConstraintActionContext)_localctx).cname = identifier(); } break; @@ -1962,11 +2070,11 @@ public AlterTableActionContext alterTableAction() { _localctx = new RenameTableActionContext(_localctx); EnterOuterAlt(_localctx, 8); { - State = 381; + State = 413; Match(RENAME); - State = 382; + State = 414; Match(TO); - State = 383; + State = 415; ((RenameTableActionContext)_localctx).newName = identifier(); } break; @@ -1974,15 +2082,15 @@ public AlterTableActionContext alterTableAction() { _localctx = new RenameColumnActionContext(_localctx); EnterOuterAlt(_localctx, 9); { - State = 384; + State = 416; Match(RENAME); - State = 385; + State = 417; Match(COLUMN); - State = 386; + State = 418; ((RenameColumnActionContext)_localctx).field = identifier(); - State = 387; + State = 419; Match(TO); - State = 388; + State = 420; ((RenameColumnActionContext)_localctx).newName = identifier(); } break; @@ -1990,15 +2098,15 @@ public AlterTableActionContext alterTableAction() { _localctx = new RenameIndexActionContext(_localctx); EnterOuterAlt(_localctx, 10); { - State = 390; + State = 422; Match(RENAME); - State = 391; + State = 423; Match(INDEX); - State = 392; + State = 424; ((RenameIndexActionContext)_localctx).index = identifier(); - State = 393; + State = 425; Match(TO); - State = 394; + State = 426; ((RenameIndexActionContext)_localctx).newName = identifier(); } break; @@ -2043,7 +2151,7 @@ public ProcedureBodyContext procedureBody() { ProcedureBodyContext _localctx = new ProcedureBodyContext(Context, State); EnterRule(_localctx, 36, RULE_procedureBody); try { - State = 401; + State = 433; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case SELECT: @@ -2051,21 +2159,21 @@ public ProcedureBodyContext procedureBody() { case LPAREN: EnterOuterAlt(_localctx, 1); { - State = 398; + State = 430; queryExpression(); } break; case INSERT: EnterOuterAlt(_localctx, 2); { - State = 399; + State = 431; insertStatement(); } break; case CREATE: EnterOuterAlt(_localctx, 3); { - State = 400; + State = 432; createTableStatement(); } break; @@ -2135,56 +2243,56 @@ public CreateIndexStatementContext createIndexStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 403; + State = 435; Match(CREATE); - State = 405; + State = 437; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==UNIQUE) { { - State = 404; + State = 436; _localctx.unique = Match(UNIQUE); } } - State = 407; + State = 439; Match(INDEX); - State = 408; + State = 440; _localctx.name = identifier(); - State = 409; + State = 441; Match(ON); - State = 410; + State = 442; _localctx.table = identifier(); - State = 411; + State = 443; Match(LPAREN); - State = 412; + State = 444; indexColumn(); - State = 417; + State = 449; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 413; + State = 445; Match(COMMA); - State = 414; + State = 446; indexColumn(); } } - State = 419; + State = 451; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 420; + State = 452; Match(RPAREN); - State = 423; + State = 455; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==WITH) { { - State = 421; + State = 453; Match(WITH); - State = 422; + State = 454; withOption(); } } @@ -2285,18 +2393,18 @@ public DropStatementContext dropStatement() { DropStatementContext _localctx = new DropStatementContext(Context, State); EnterRule(_localctx, 40, RULE_dropStatement); try { - State = 440; + State = 472; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,35,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,37,Context) ) { case 1: _localctx = new DropTableStatementContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 425; + State = 457; Match(DROP); - State = 426; + State = 458; Match(TABLE); - State = 427; + State = 459; ((DropTableStatementContext)_localctx).table = identifier(); } break; @@ -2304,15 +2412,15 @@ public DropStatementContext dropStatement() { _localctx = new DropIndexStatementContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 428; + State = 460; Match(DROP); - State = 429; + State = 461; Match(INDEX); - State = 430; + State = 462; ((DropIndexStatementContext)_localctx).index = identifier(); - State = 431; + State = 463; Match(ON); - State = 432; + State = 464; ((DropIndexStatementContext)_localctx).table = identifier(); } break; @@ -2320,11 +2428,11 @@ public DropStatementContext dropStatement() { _localctx = new DropProcedureStatementContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 434; + State = 466; Match(DROP); - State = 435; + State = 467; Match(PROCEDURE); - State = 436; + State = 468; ((DropProcedureStatementContext)_localctx).proc = identifier(); } break; @@ -2332,11 +2440,11 @@ public DropStatementContext dropStatement() { _localctx = new DropViewStatementContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 437; + State = 469; Match(DROP); - State = 438; + State = 470; Match(VIEW); - State = 439; + State = 471; ((DropViewStatementContext)_localctx).view = identifier(); } break; @@ -2382,14 +2490,14 @@ public IndexColumnContext indexColumn() { try { EnterOuterAlt(_localctx, 1); { - State = 442; + State = 474; _localctx.col = identifier(); - State = 444; + State = 476; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ASC || _la==DESC) { { - State = 443; + State = 475; _localctx.dir = TokenStream.LT(1); _la = TokenStream.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -2465,14 +2573,14 @@ public WithOptionContext withOption() { WithOptionContext _localctx = new WithOptionContext(Context, State); EnterRule(_localctx, 44, RULE_withOption); try { - State = 451; + State = 483; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case PRIMARY: _localctx = new WithPrimaryContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 446; + State = 478; Match(PRIMARY); } break; @@ -2480,9 +2588,9 @@ public WithOptionContext withOption() { _localctx = new WithDisallowNullContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 447; + State = 479; Match(DISALLOW); - State = 448; + State = 480; Match(NULL); } break; @@ -2490,9 +2598,9 @@ public WithOptionContext withOption() { _localctx = new WithIgnoreNullContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 449; + State = 481; Match(IGNORE); - State = 450; + State = 482; Match(NULL); } break; @@ -2549,31 +2657,31 @@ public ColumnDefinitionContext columnDefinition() { try { EnterOuterAlt(_localctx, 1); { - State = 453; + State = 485; _localctx.name = identifier(); - State = 454; + State = 486; dataType(); - State = 456; + State = 488; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==AS) { { - State = 455; + State = 487; calculatedClause(); } } - State = 461; + State = 493; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - while (_la==NOT || ((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & 2149656597L) != 0)) { + while (_la==NOT || ((((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 17197213717L) != 0)) { { { - State = 458; + State = 490; columnConstraint(); } } - State = 463; + State = 495; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -2617,13 +2725,13 @@ public CalculatedClauseContext calculatedClause() { try { EnterOuterAlt(_localctx, 1); { - State = 464; + State = 496; Match(AS); - State = 465; + State = 497; Match(LPAREN); - State = 466; + State = 498; expression(0); - State = 467; + State = 499; Match(RPAREN); } } @@ -2640,6 +2748,7 @@ public CalculatedClauseContext calculatedClause() { public partial class DataTypeContext : ParserRuleContext { public IdentifierContext typeName; + public IToken identityType; public IdentifierContext extra; public IdentifierContext extra2; public SignedIntegerContext size; @@ -2650,6 +2759,7 @@ [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext[] identifier() [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext identifier(int i) { return GetRuleContext(i); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode IDENTITY() { return GetToken(AccessSqlParser.IDENTITY, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LPAREN() { return GetToken(AccessSqlParser.LPAREN, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RPAREN() { return GetToken(AccessSqlParser.RPAREN, 0); } [System.Diagnostics.DebuggerNonUserCode] public SignedIntegerContext[] signedInteger() { @@ -2680,50 +2790,82 @@ public DataTypeContext dataType() { try { EnterOuterAlt(_localctx, 1); { - State = 469; - _localctx.typeName = identifier(); - State = 471; + State = 503; + ErrorHandler.Sync(this); + switch (TokenStream.LA(1)) { + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + case BRACKET_ID: + case BACKTICK_ID: + case IDENTIFIER: + { + State = 501; + _localctx.typeName = identifier(); + } + break; + case IDENTITY: + { + State = 502; + _localctx.identityType = Match(IDENTITY); + } + break; + default: + throw new NoViableAltException(this); + } + State = 506; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,40,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,43,Context) ) { case 1: { - State = 470; + State = 505; _localctx.extra = identifier(); } break; } - State = 474; + State = 509; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) { + if (((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7696581410815L) != 0)) { { - State = 473; + State = 508; _localctx.extra2 = identifier(); } } - State = 484; + State = 519; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==LPAREN) { { - State = 476; + State = 511; Match(LPAREN); - State = 477; + State = 512; _localctx.size = signedInteger(); - State = 480; + State = 515; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==COMMA) { { - State = 478; + State = 513; Match(COMMA); - State = 479; + State = 514; _localctx.scale = signedInteger(); } } - State = 482; + State = 517; Match(RPAREN); } } @@ -2765,17 +2907,17 @@ public SignedIntegerContext signedInteger() { try { EnterOuterAlt(_localctx, 1); { - State = 487; + State = 522; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==MINUS) { { - State = 486; + State = 521; Match(MINUS); } } - State = 489; + State = 524; Match(INTEGER_LITERAL); } } @@ -2890,6 +3032,27 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } + public partial class IdentityConstraintContext : ColumnConstraintContext { + public SignedIntegerContext seed; + public SignedIntegerContext increment; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode IDENTITY() { return GetToken(AccessSqlParser.IDENTITY, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LPAREN() { return GetToken(AccessSqlParser.LPAREN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RPAREN() { return GetToken(AccessSqlParser.RPAREN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public SignedIntegerContext[] signedInteger() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public SignedIntegerContext signedInteger(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode COMMA() { return GetToken(AccessSqlParser.COMMA, 0); } + public IdentityConstraintContext(ColumnConstraintContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitIdentityConstraint(this); + else return visitor.VisitChildren(this); + } + } public partial class NotNullConstraintContext : ColumnConstraintContext { [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NOT() { return GetToken(AccessSqlParser.NOT, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NULL() { return GetToken(AccessSqlParser.NULL, 0); } @@ -2906,6 +3069,9 @@ public partial class PrimaryKeyConstraintContext : ColumnConstraintContext { [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode PRIMARY() { return GetToken(AccessSqlParser.PRIMARY, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode KEY() { return GetToken(AccessSqlParser.KEY, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CONSTRAINT() { return GetToken(AccessSqlParser.CONSTRAINT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ClusteredOptionContext clusteredOption() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext identifier() { return GetRuleContext(0); } @@ -2921,6 +3087,9 @@ public partial class UniqueColumnConstraintContext : ColumnConstraintContext { public IdentifierContext cname; [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode UNIQUE() { return GetToken(AccessSqlParser.UNIQUE, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CONSTRAINT() { return GetToken(AccessSqlParser.CONSTRAINT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ClusteredOptionContext clusteredOption() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext identifier() { return GetRuleContext(0); } @@ -2939,16 +3108,16 @@ public ColumnConstraintContext columnConstraint() { EnterRule(_localctx, 54, RULE_columnConstraint); int _la; try { - State = 543; + State = 595; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,52,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,59,Context) ) { case 1: _localctx = new NotNullConstraintContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 491; + State = 526; Match(NOT); - State = 492; + State = 527; Match(NULL); } break; @@ -2956,7 +3125,7 @@ public ColumnConstraintContext columnConstraint() { _localctx = new NullableConstraintContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 493; + State = 528; Match(NULL); } break; @@ -2964,9 +3133,9 @@ public ColumnConstraintContext columnConstraint() { _localctx = new DefaultConstraintContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 494; + State = 529; Match(DEFAULT); - State = 495; + State = 530; expression(0); } break; @@ -2974,9 +3143,9 @@ public ColumnConstraintContext columnConstraint() { _localctx = new CompressionConstraintContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 496; + State = 531; Match(WITH); - State = 497; + State = 532; _la = TokenStream.LA(1); if ( !(_la==COMPRESSION || _la==COMP) ) { ErrorHandler.RecoverInline(this); @@ -2991,25 +3160,25 @@ public ColumnConstraintContext columnConstraint() { _localctx = new CheckColumnConstraintContext(_localctx); EnterOuterAlt(_localctx, 5); { - State = 500; + State = 535; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 498; + State = 533; Match(CONSTRAINT); - State = 499; + State = 534; ((CheckColumnConstraintContext)_localctx).cname = identifier(); } } - State = 502; + State = 537; Match(CHECK); - State = 503; + State = 538; Match(LPAREN); - State = 504; + State = 539; checkBody(); - State = 505; + State = 540; Match(RPAREN); } break; @@ -3017,110 +3186,164 @@ public ColumnConstraintContext columnConstraint() { _localctx = new PrimaryKeyConstraintContext(_localctx); EnterOuterAlt(_localctx, 6); { - State = 509; + State = 544; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 507; + State = 542; Match(CONSTRAINT); - State = 508; + State = 543; ((PrimaryKeyConstraintContext)_localctx).cname = identifier(); } } - State = 511; + State = 546; Match(PRIMARY); - State = 512; + State = 547; Match(KEY); + State = 549; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==CLUSTERED || _la==NONCLUSTERED) { + { + State = 548; + clusteredOption(); + } + } + } break; case 7: _localctx = new UniqueColumnConstraintContext(_localctx); EnterOuterAlt(_localctx, 7); { - State = 515; + State = 553; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 513; + State = 551; Match(CONSTRAINT); - State = 514; + State = 552; ((UniqueColumnConstraintContext)_localctx).cname = identifier(); } } - State = 517; + State = 555; Match(UNIQUE); + State = 557; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==CLUSTERED || _la==NONCLUSTERED) { + { + State = 556; + clusteredOption(); + } } - break; + + } + break; case 8: _localctx = new ColumnReferencesConstraintContext(_localctx); EnterOuterAlt(_localctx, 8); { - State = 520; + State = 561; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 518; + State = 559; Match(CONSTRAINT); - State = 519; + State = 560; ((ColumnReferencesConstraintContext)_localctx).cname = identifier(); } } - State = 522; + State = 563; Match(REFERENCES); - State = 523; + State = 564; ((ColumnReferencesConstraintContext)_localctx).refTable = identifier(); - State = 535; + State = 576; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==LPAREN) { { - State = 524; + State = 565; Match(LPAREN); - State = 525; + State = 566; ((ColumnReferencesConstraintContext)_localctx)._identifier = identifier(); ((ColumnReferencesConstraintContext)_localctx)._refColumns.Add(((ColumnReferencesConstraintContext)_localctx)._identifier); - State = 530; + State = 571; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 526; + State = 567; Match(COMMA); - State = 527; + State = 568; ((ColumnReferencesConstraintContext)_localctx)._identifier = identifier(); ((ColumnReferencesConstraintContext)_localctx)._refColumns.Add(((ColumnReferencesConstraintContext)_localctx)._identifier); } } - State = 532; + State = 573; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 533; + State = 574; Match(RPAREN); } } - State = 540; + State = 581; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==ON) { { { - State = 537; + State = 578; foreignKeyAction(); } } - State = 542; + State = 583; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + } + } + break; + case 9: + _localctx = new IdentityConstraintContext(_localctx); + EnterOuterAlt(_localctx, 9); + { + State = 584; + Match(IDENTITY); + State = 593; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==LPAREN) { + { + State = 585; + Match(LPAREN); + State = 586; + ((IdentityConstraintContext)_localctx).seed = signedInteger(); + State = 589; ErrorHandler.Sync(this); _la = TokenStream.LA(1); + if (_la==COMMA) { + { + State = 587; + Match(COMMA); + State = 588; + ((IdentityConstraintContext)_localctx).increment = signedInteger(); + } + } + + State = 591; + Match(RPAREN); + } } + } break; } @@ -3163,6 +3386,9 @@ [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext identifier(int return GetRuleContext(i); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CONSTRAINT() { return GetToken(AccessSqlParser.CONSTRAINT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ClusteredOptionContext clusteredOption() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode[] COMMA() { return GetTokens(AccessSqlParser.COMMA); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode COMMA(int i) { return GetToken(AccessSqlParser.COMMA, i); @@ -3234,6 +3460,9 @@ [System.Diagnostics.DebuggerNonUserCode] public IdentifierContext identifier(int return GetRuleContext(i); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CONSTRAINT() { return GetToken(AccessSqlParser.CONSTRAINT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ClusteredOptionContext clusteredOption() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode[] COMMA() { return GetTokens(AccessSqlParser.COMMA); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode COMMA(int i) { return GetToken(AccessSqlParser.COMMA, i); @@ -3273,52 +3502,62 @@ public TableConstraintContext tableConstraint() { EnterRule(_localctx, 56, RULE_tableConstraint); int _la; try { - State = 628; + State = 686; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,64,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,73,Context) ) { case 1: _localctx = new PrimaryKeyTableConstraintContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 547; + State = 599; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 545; + State = 597; Match(CONSTRAINT); - State = 546; + State = 598; ((PrimaryKeyTableConstraintContext)_localctx).name = identifier(); } } - State = 549; + State = 601; Match(PRIMARY); - State = 550; + State = 602; Match(KEY); - State = 551; + State = 604; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==CLUSTERED || _la==NONCLUSTERED) { + { + State = 603; + clusteredOption(); + } + } + + State = 606; Match(LPAREN); - State = 552; + State = 607; ((PrimaryKeyTableConstraintContext)_localctx)._identifier = identifier(); ((PrimaryKeyTableConstraintContext)_localctx)._columns.Add(((PrimaryKeyTableConstraintContext)_localctx)._identifier); - State = 557; + State = 612; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 553; + State = 608; Match(COMMA); - State = 554; + State = 609; ((PrimaryKeyTableConstraintContext)_localctx)._identifier = identifier(); ((PrimaryKeyTableConstraintContext)_localctx)._columns.Add(((PrimaryKeyTableConstraintContext)_localctx)._identifier); } } - State = 559; + State = 614; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 560; + State = 615; Match(RPAREN); } break; @@ -3326,43 +3565,53 @@ public TableConstraintContext tableConstraint() { _localctx = new UniqueTableConstraintContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 564; + State = 619; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 562; + State = 617; Match(CONSTRAINT); - State = 563; + State = 618; ((UniqueTableConstraintContext)_localctx).name = identifier(); } } - State = 566; + State = 621; Match(UNIQUE); - State = 567; + State = 623; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==CLUSTERED || _la==NONCLUSTERED) { + { + State = 622; + clusteredOption(); + } + } + + State = 625; Match(LPAREN); - State = 568; + State = 626; ((UniqueTableConstraintContext)_localctx)._identifier = identifier(); ((UniqueTableConstraintContext)_localctx)._columns.Add(((UniqueTableConstraintContext)_localctx)._identifier); - State = 573; + State = 631; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 569; + State = 627; Match(COMMA); - State = 570; + State = 628; ((UniqueTableConstraintContext)_localctx)._identifier = identifier(); ((UniqueTableConstraintContext)_localctx)._columns.Add(((UniqueTableConstraintContext)_localctx)._identifier); } } - State = 575; + State = 633; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 576; + State = 634; Match(RPAREN); } break; @@ -3370,105 +3619,105 @@ public TableConstraintContext tableConstraint() { _localctx = new ForeignKeyTableConstraintContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 580; + State = 638; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 578; + State = 636; Match(CONSTRAINT); - State = 579; + State = 637; ((ForeignKeyTableConstraintContext)_localctx).name = identifier(); } } - State = 582; + State = 640; Match(FOREIGN); - State = 583; + State = 641; Match(KEY); - State = 586; + State = 644; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NO) { { - State = 584; + State = 642; ((ForeignKeyTableConstraintContext)_localctx).noIndex = Match(NO); - State = 585; + State = 643; Match(INDEX); } } - State = 588; + State = 646; Match(LPAREN); - State = 589; + State = 647; ((ForeignKeyTableConstraintContext)_localctx)._identifier = identifier(); ((ForeignKeyTableConstraintContext)_localctx)._columns.Add(((ForeignKeyTableConstraintContext)_localctx)._identifier); - State = 594; + State = 652; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 590; + State = 648; Match(COMMA); - State = 591; + State = 649; ((ForeignKeyTableConstraintContext)_localctx)._identifier = identifier(); ((ForeignKeyTableConstraintContext)_localctx)._columns.Add(((ForeignKeyTableConstraintContext)_localctx)._identifier); } } - State = 596; + State = 654; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 597; + State = 655; Match(RPAREN); - State = 598; + State = 656; Match(REFERENCES); - State = 599; + State = 657; ((ForeignKeyTableConstraintContext)_localctx).refTable = identifier(); - State = 611; + State = 669; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==LPAREN) { { - State = 600; + State = 658; Match(LPAREN); - State = 601; + State = 659; ((ForeignKeyTableConstraintContext)_localctx)._identifier = identifier(); ((ForeignKeyTableConstraintContext)_localctx)._refColumns.Add(((ForeignKeyTableConstraintContext)_localctx)._identifier); - State = 606; + State = 664; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 602; + State = 660; Match(COMMA); - State = 603; + State = 661; ((ForeignKeyTableConstraintContext)_localctx)._identifier = identifier(); ((ForeignKeyTableConstraintContext)_localctx)._refColumns.Add(((ForeignKeyTableConstraintContext)_localctx)._identifier); } } - State = 608; + State = 666; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 609; + State = 667; Match(RPAREN); } } - State = 616; + State = 674; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==ON) { { { - State = 613; + State = 671; foreignKeyAction(); } } - State = 618; + State = 676; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -3478,25 +3727,25 @@ public TableConstraintContext tableConstraint() { _localctx = new CheckTableConstraintContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 621; + State = 679; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==CONSTRAINT) { { - State = 619; + State = 677; Match(CONSTRAINT); - State = 620; + State = 678; ((CheckTableConstraintContext)_localctx).name = identifier(); } } - State = 623; + State = 681; Match(CHECK); - State = 624; + State = 682; Match(LPAREN); - State = 625; + State = 683; checkBody(); - State = 626; + State = 684; Match(RPAREN); } break; @@ -3549,12 +3798,12 @@ public CheckBodyContext checkBody() { try { EnterOuterAlt(_localctx, 1); { - State = 637; + State = 695; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - while ((((_la) & ~0x3f) == 0 && ((1L << _la) & -2L) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & -4503599627370497L) != 0) || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 63L) != 0)) { + while ((((_la) & ~0x3f) == 0 && ((1L << _la) & -2L) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & -1L) != 0) || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 67108607L) != 0)) { { - State = 635; + State = 693; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case SELECT: @@ -3565,6 +3814,9 @@ public CheckBodyContext checkBody() { case AND: case OR: case NOT: + case XOR: + case EQV: + case IMP: case BAND: case BOR: case BXOR: @@ -3640,6 +3892,9 @@ public CheckBodyContext checkBody() { case DEFAULT: case NO: case UNIQUE: + case CLUSTERED: + case IDENTITY: + case NONCLUSTERED: case INDEX: case TEMPORARY: case WITH: @@ -3658,6 +3913,20 @@ public CheckBodyContext checkBody() { case TRUE: case FALSE: case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case STAR: case SLASH: case BACKSLASH: @@ -3689,7 +3958,7 @@ public CheckBodyContext checkBody() { case LINE_COMMENT: case BLOCK_COMMENT: { - State = 630; + State = 688; _la = TokenStream.LA(1); if ( _la <= 0 || (_la==LPAREN || _la==RPAREN) ) { ErrorHandler.RecoverInline(this); @@ -3702,11 +3971,11 @@ public CheckBodyContext checkBody() { break; case LPAREN: { - State = 631; + State = 689; Match(LPAREN); - State = 632; + State = 690; checkBody(); - State = 633; + State = 691; Match(RPAREN); } break; @@ -3714,7 +3983,7 @@ public CheckBodyContext checkBody() { throw new NoViableAltException(this); } } - State = 639; + State = 697; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -3777,18 +4046,18 @@ public ForeignKeyActionContext foreignKeyAction() { ForeignKeyActionContext _localctx = new ForeignKeyActionContext(Context, State); EnterRule(_localctx, 60, RULE_foreignKeyAction); try { - State = 646; + State = 704; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,67,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,76,Context) ) { case 1: _localctx = new OnUpdateActionContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 640; + State = 698; Match(ON); - State = 641; + State = 699; Match(UPDATE); - State = 642; + State = 700; referentialAction(); } break; @@ -3796,11 +4065,11 @@ public ForeignKeyActionContext foreignKeyAction() { _localctx = new OnDeleteActionContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 643; + State = 701; Match(ON); - State = 644; + State = 702; Match(DELETE); - State = 645; + State = 703; referentialAction(); } break; @@ -3888,14 +4157,14 @@ public ReferentialActionContext referentialAction() { ReferentialActionContext _localctx = new ReferentialActionContext(Context, State); EnterRule(_localctx, 62, RULE_referentialAction); try { - State = 656; + State = 714; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,68,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,77,Context) ) { case 1: _localctx = new CascadeActionContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 648; + State = 706; Match(CASCADE); } break; @@ -3903,9 +4172,9 @@ public ReferentialActionContext referentialAction() { _localctx = new NoActionActionContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 649; + State = 707; Match(NO); - State = 650; + State = 708; Match(ACTION); } break; @@ -3913,7 +4182,7 @@ public ReferentialActionContext referentialAction() { _localctx = new RestrictActionContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 651; + State = 709; Match(RESTRICT); } break; @@ -3921,9 +4190,9 @@ public ReferentialActionContext referentialAction() { _localctx = new SetNullActionContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 652; + State = 710; Match(SET); - State = 653; + State = 711; Match(NULL); } break; @@ -3931,9 +4200,9 @@ public ReferentialActionContext referentialAction() { _localctx = new SetDefaultActionContext(_localctx); EnterOuterAlt(_localctx, 5); { - State = 654; + State = 712; Match(SET); - State = 655; + State = 713; Match(DEFAULT); } break; @@ -4001,73 +4270,73 @@ public InsertStatementContext insertStatement() { try { EnterOuterAlt(_localctx, 1); { - State = 658; + State = 716; Match(INSERT); - State = 659; + State = 717; Match(INTO); - State = 660; + State = 718; _localctx.table = identifier(); - State = 688; + State = 746; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case SELECT: case VALUES: case LPAREN: { - State = 672; + State = 730; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,70,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,79,Context) ) { case 1: { - State = 661; + State = 719; Match(LPAREN); - State = 662; + State = 720; _localctx._identifier = identifier(); _localctx._columns.Add(_localctx._identifier); - State = 667; + State = 725; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 663; + State = 721; Match(COMMA); - State = 664; + State = 722; _localctx._identifier = identifier(); _localctx._columns.Add(_localctx._identifier); } } - State = 669; + State = 727; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 670; + State = 728; Match(RPAREN); } break; } - State = 684; + State = 742; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,72,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,81,Context) ) { case 1: { - State = 674; + State = 732; Match(VALUES); - State = 675; + State = 733; rowValues(); - State = 680; + State = 738; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 676; + State = 734; Match(COMMA); - State = 677; + State = 735; rowValues(); } } - State = 682; + State = 740; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -4075,7 +4344,7 @@ public InsertStatementContext insertStatement() { break; case 2: { - State = 683; + State = 741; _localctx.source = queryExpression(); } break; @@ -4084,9 +4353,9 @@ public InsertStatementContext insertStatement() { break; case DEFAULT: { - State = 686; + State = 744; Match(DEFAULT); - State = 687; + State = 745; Match(VALUES); } break; @@ -4140,27 +4409,27 @@ public RowValuesContext rowValues() { try { EnterOuterAlt(_localctx, 1); { - State = 690; + State = 748; Match(LPAREN); - State = 691; + State = 749; rowValue(); - State = 696; + State = 754; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 692; + State = 750; Match(COMMA); - State = 693; + State = 751; rowValue(); } } - State = 698; + State = 756; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 699; + State = 757; Match(RPAREN); } } @@ -4198,13 +4467,13 @@ public RowValueContext rowValue() { RowValueContext _localctx = new RowValueContext(Context, State); EnterRule(_localctx, 68, RULE_rowValue); try { - State = 703; + State = 761; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case DEFAULT: EnterOuterAlt(_localctx, 1); { - State = 701; + State = 759; Match(DEFAULT); } break; @@ -4220,6 +4489,21 @@ public RowValueContext rowValue() { case TRUE: case FALSE: case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + case PLUS: case MINUS: case LPAREN: case SYSVAR: @@ -4235,7 +4519,7 @@ public RowValueContext rowValue() { case IDENTIFIER: EnterOuterAlt(_localctx, 2); { - State = 702; + State = 760; expression(0); } break; @@ -4294,40 +4578,40 @@ public QueryExpressionContext queryExpression() { try { EnterOuterAlt(_localctx, 1); { - State = 705; + State = 763; queryTerm(); - State = 711; + State = 769; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - while ((((_la) & ~0x3f) == 0 && ((1L << _la) & 14636698788954112L) != 0)) { + while ((((_la) & ~0x3f) == 0 && ((1L << _la) & 117093590311632896L) != 0)) { { { - State = 706; + State = 764; setOperator(); - State = 707; + State = 765; queryTerm(); } } - State = 713; + State = 771; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 715; + State = 773; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ORDER) { { - State = 714; + State = 772; orderByClause(); } } - State = 718; + State = 776; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==OFFSET || _la==FETCH) { { - State = 717; + State = 775; offsetFetchClause(); } } @@ -4410,14 +4694,14 @@ public QueryTermContext queryTerm() { EnterRule(_localctx, 72, RULE_queryTerm); int _la; try { - State = 734; + State = 792; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case SELECT: _localctx = new SelectTermContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 720; + State = 778; querySpecification(); } break; @@ -4425,11 +4709,11 @@ public QueryTermContext queryTerm() { _localctx = new ParenTermContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 721; + State = 779; Match(LPAREN); - State = 722; + State = 780; queryExpression(); - State = 723; + State = 781; Match(RPAREN); } break; @@ -4437,23 +4721,23 @@ public QueryTermContext queryTerm() { _localctx = new ValuesTermContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 725; + State = 783; Match(VALUES); - State = 726; + State = 784; rowValues(); - State = 731; + State = 789; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 727; + State = 785; Match(COMMA); - State = 728; + State = 786; rowValues(); } } - State = 733; + State = 791; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -4498,20 +4782,20 @@ public SetOperatorContext setOperator() { EnterRule(_localctx, 74, RULE_setOperator); int _la; try { - State = 742; + State = 800; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case UNION: EnterOuterAlt(_localctx, 1); { - State = 736; + State = 794; Match(UNION); - State = 738; + State = 796; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ALL) { { - State = 737; + State = 795; Match(ALL); } } @@ -4521,14 +4805,14 @@ public SetOperatorContext setOperator() { case INTERSECT: EnterOuterAlt(_localctx, 2); { - State = 740; + State = 798; Match(INTERSECT); } break; case EXCEPT: EnterOuterAlt(_localctx, 3); { - State = 741; + State = 799; Match(EXCEPT); } break; @@ -4597,78 +4881,78 @@ public QuerySpecificationContext querySpecification() { try { EnterOuterAlt(_localctx, 1); { - State = 744; + State = 802; Match(SELECT); - State = 746; + State = 804; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 2251806256136192L) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 18014450049089536L) != 0)) { { - State = 745; + State = 803; _localctx.predicate = selectPredicate(); } } - State = 749; + State = 807; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==TOP) { { - State = 748; + State = 806; topClause(); } } - State = 751; + State = 809; selectList(); - State = 754; + State = 812; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==INTO) { { - State = 752; + State = 810; Match(INTO); - State = 753; + State = 811; _localctx.into = identifier(); } } - State = 757; + State = 815; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==FROM) { { - State = 756; + State = 814; fromClause(); } } - State = 760; + State = 818; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==WHERE) { { - State = 759; + State = 817; whereClause(); } } - State = 763; + State = 821; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==GROUP) { { - State = 762; + State = 820; groupByClause(); } } - State = 766; + State = 824; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==HAVING) { { - State = 765; + State = 823; havingClause(); } } @@ -4711,9 +4995,9 @@ public SelectPredicateContext selectPredicate() { try { EnterOuterAlt(_localctx, 1); { - State = 768; + State = 826; _la = TokenStream.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 2251806256136192L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 18014450049089536L) != 0)) ) { ErrorHandler.RecoverInline(this); } else { @@ -4767,25 +5051,25 @@ public GroupByClauseContext groupByClause() { try { EnterOuterAlt(_localctx, 1); { - State = 770; + State = 828; Match(GROUP); - State = 771; + State = 829; Match(BY); - State = 772; + State = 830; expression(0); - State = 777; + State = 835; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 773; + State = 831; Match(COMMA); - State = 774; + State = 832; expression(0); } } - State = 779; + State = 837; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -4827,9 +5111,9 @@ public HavingClauseContext havingClause() { try { EnterOuterAlt(_localctx, 1); { - State = 780; + State = 838; Match(HAVING); - State = 781; + State = 839; expression(0); } } @@ -4884,18 +5168,18 @@ public TopClauseContext topClause() { int _alt; EnterOuterAlt(_localctx, 1); { - State = 783; + State = 841; Match(TOP); - State = 784; + State = 842; topOperand(); - State = 789; + State = 847; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,91,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,100,Context); while ( _alt!=2 && _alt!=global::Antlr4.Runtime.Atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - State = 785; + State = 843; _la = TokenStream.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { ErrorHandler.RecoverInline(this); @@ -4904,21 +5188,21 @@ public TopClauseContext topClause() { ErrorHandler.ReportMatch(this); Consume(); } - State = 786; + State = 844; topOperand(); } } } - State = 791; + State = 849; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,91,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,100,Context); } - State = 793; + State = 851; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==PERCENT) { { - State = 792; + State = 850; _localctx.percent = Match(PERCENT); } } @@ -4962,31 +5246,31 @@ public TopOperandContext topOperand() { TopOperandContext _localctx = new TopOperandContext(Context, State); EnterRule(_localctx, 86, RULE_topOperand); try { - State = 801; + State = 859; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case INTEGER_LITERAL: EnterOuterAlt(_localctx, 1); { - State = 795; + State = 853; Match(INTEGER_LITERAL); } break; case PARAM: EnterOuterAlt(_localctx, 2); { - State = 796; + State = 854; Match(PARAM); } break; case LPAREN: EnterOuterAlt(_localctx, 3); { - State = 797; + State = 855; Match(LPAREN); - State = 798; + State = 856; expression(0); - State = 799; + State = 857; Match(RPAREN); } break; @@ -5044,26 +5328,26 @@ public OffsetFetchClauseContext offsetFetchClause() { EnterRule(_localctx, 88, RULE_offsetFetchClause); int _la; try { - State = 820; + State = 878; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case OFFSET: EnterOuterAlt(_localctx, 1); { - State = 803; + State = 861; Match(OFFSET); - State = 804; + State = 862; _localctx.offset = expression(0); - State = 805; + State = 863; rowKeyword(); - State = 812; + State = 870; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==FETCH) { { - State = 806; + State = 864; Match(FETCH); - State = 807; + State = 865; _la = TokenStream.LA(1); if ( !(_la==NEXT || _la==FIRST) ) { ErrorHandler.RecoverInline(this); @@ -5072,11 +5356,11 @@ public OffsetFetchClauseContext offsetFetchClause() { ErrorHandler.ReportMatch(this); Consume(); } - State = 808; + State = 866; _localctx.limit = expression(0); - State = 809; + State = 867; rowKeyword(); - State = 810; + State = 868; Match(ONLY); } } @@ -5086,9 +5370,9 @@ public OffsetFetchClauseContext offsetFetchClause() { case FETCH: EnterOuterAlt(_localctx, 2); { - State = 814; + State = 872; Match(FETCH); - State = 815; + State = 873; _la = TokenStream.LA(1); if ( !(_la==NEXT || _la==FIRST) ) { ErrorHandler.RecoverInline(this); @@ -5097,11 +5381,11 @@ public OffsetFetchClauseContext offsetFetchClause() { ErrorHandler.ReportMatch(this); Consume(); } - State = 816; + State = 874; _localctx.limit = expression(0); - State = 817; + State = 875; rowKeyword(); - State = 818; + State = 876; Match(ONLY); } break; @@ -5144,7 +5428,7 @@ public RowKeywordContext rowKeyword() { try { EnterOuterAlt(_localctx, 1); { - State = 822; + State = 880; _la = TokenStream.LA(1); if ( !(_la==ROWS || _la==ROW) ) { ErrorHandler.RecoverInline(this); @@ -5197,13 +5481,13 @@ public SelectListContext selectList() { EnterRule(_localctx, 92, RULE_selectList); int _la; try { - State = 833; + State = 891; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case STAR: EnterOuterAlt(_localctx, 1); { - State = 824; + State = 882; Match(STAR); } break; @@ -5219,6 +5503,21 @@ public SelectListContext selectList() { case TRUE: case FALSE: case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + case PLUS: case MINUS: case LPAREN: case SYSVAR: @@ -5234,21 +5533,21 @@ public SelectListContext selectList() { case IDENTIFIER: EnterOuterAlt(_localctx, 2); { - State = 825; + State = 883; selectItem(); - State = 830; + State = 888; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 826; + State = 884; Match(COMMA); - State = 827; + State = 885; selectItem(); } } - State = 832; + State = 890; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -5320,18 +5619,18 @@ public SelectItemContext selectItem() { EnterRule(_localctx, 94, RULE_selectItem); int _la; try { - State = 846; + State = 904; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,100,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,109,Context) ) { case 1: _localctx = new QualifiedStarSelectItemContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 835; + State = 893; ((QualifiedStarSelectItemContext)_localctx).qualifier = identifier(); - State = 836; + State = 894; Match(DOT); - State = 837; + State = 895; Match(STAR); } break; @@ -5339,24 +5638,24 @@ public SelectItemContext selectItem() { _localctx = new ExpressionSelectItemContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 839; + State = 897; expression(0); - State = 844; + State = 902; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (_la==AS || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) { + if (_la==AS || ((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7696581410815L) != 0)) { { - State = 841; + State = 899; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==AS) { { - State = 840; + State = 898; Match(AS); } } - State = 843; + State = 901; ((ExpressionSelectItemContext)_localctx).alias = identifier(); } } @@ -5409,23 +5708,23 @@ public FromClauseContext fromClause() { try { EnterOuterAlt(_localctx, 1); { - State = 848; + State = 906; Match(FROM); - State = 849; + State = 907; tableSource(); - State = 854; + State = 912; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 850; + State = 908; Match(COMMA); - State = 851; + State = 909; tableSource(); } } - State = 856; + State = 914; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -5473,19 +5772,19 @@ public TableSourceContext tableSource() { try { EnterOuterAlt(_localctx, 1); { - State = 857; + State = 915; tablePrimary(); - State = 861; + State = 919; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - while ((((_la) & ~0x3f) == 0 && ((1L << _la) & 17181933568L) != 0)) { + while ((((_la) & ~0x3f) == 0 && ((1L << _la) & 137455468544L) != 0)) { { { - State = 858; + State = 916; joinClause(); } } - State = 863; + State = 921; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -5572,31 +5871,31 @@ public TablePrimaryContext tablePrimary() { EnterRule(_localctx, 100, RULE_tablePrimary); int _la; try { - State = 884; + State = 942; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,107,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,116,Context) ) { case 1: _localctx = new NamedTablePrimaryContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 864; + State = 922; ((NamedTablePrimaryContext)_localctx).table = identifier(); - State = 869; + State = 927; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (_la==AS || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) { + if (_la==AS || ((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7696581410815L) != 0)) { { - State = 866; + State = 924; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==AS) { { - State = 865; + State = 923; Match(AS); } } - State = 868; + State = 926; ((NamedTablePrimaryContext)_localctx).alias = identifier(); } } @@ -5607,28 +5906,28 @@ public TablePrimaryContext tablePrimary() { _localctx = new SubqueryPrimaryContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 871; + State = 929; Match(LPAREN); - State = 872; + State = 930; queryExpression(); - State = 873; + State = 931; Match(RPAREN); - State = 878; + State = 936; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if (_la==AS || ((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) { + if (_la==AS || ((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 7696581410815L) != 0)) { { - State = 875; + State = 933; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==AS) { { - State = 874; + State = 932; Match(AS); } } - State = 877; + State = 935; ((SubqueryPrimaryContext)_localctx).alias = identifier(); } } @@ -5639,11 +5938,11 @@ public TablePrimaryContext tablePrimary() { _localctx = new ParenJoinPrimaryContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 880; + State = 938; Match(LPAREN); - State = 881; + State = 939; tableSource(); - State = 882; + State = 940; Match(RPAREN); } break; @@ -5740,22 +6039,22 @@ public JoinClauseContext joinClause() { JoinClauseContext _localctx = new JoinClauseContext(Context, State); EnterRule(_localctx, 102, RULE_joinClause); try { - State = 901; + State = 959; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,108,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,117,Context) ) { case 1: _localctx = new ConditionalJoinContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 886; + State = 944; joinType(); - State = 887; + State = 945; Match(JOIN); - State = 888; + State = 946; tablePrimary(); - State = 889; + State = 947; Match(ON); - State = 890; + State = 948; expression(0); } break; @@ -5763,11 +6062,11 @@ public JoinClauseContext joinClause() { _localctx = new CrossJoinContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 892; + State = 950; Match(CROSS); - State = 893; + State = 951; Match(JOIN); - State = 894; + State = 952; tablePrimary(); } break; @@ -5775,11 +6074,11 @@ public JoinClauseContext joinClause() { _localctx = new CrossApplyContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 895; + State = 953; Match(CROSS); - State = 896; + State = 954; Match(APPLY); - State = 897; + State = 955; tablePrimary(); } break; @@ -5787,11 +6086,11 @@ public JoinClauseContext joinClause() { _localctx = new OuterApplyContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 898; + State = 956; Match(OUTER); - State = 899; + State = 957; Match(APPLY); - State = 900; + State = 958; tablePrimary(); } break; @@ -5870,7 +6169,7 @@ public JoinTypeContext joinType() { EnterRule(_localctx, 104, RULE_joinType); int _la; try { - State = 918; + State = 976; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case INNER: @@ -5878,12 +6177,12 @@ public JoinTypeContext joinType() { _localctx = new InnerJoinContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 904; + State = 962; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==INNER) { { - State = 903; + State = 961; Match(INNER); } } @@ -5894,14 +6193,14 @@ public JoinTypeContext joinType() { _localctx = new LeftJoinContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 906; + State = 964; Match(LEFT); - State = 908; + State = 966; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==OUTER) { { - State = 907; + State = 965; Match(OUTER); } } @@ -5912,14 +6211,14 @@ public JoinTypeContext joinType() { _localctx = new RightJoinContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 910; + State = 968; Match(RIGHT); - State = 912; + State = 970; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==OUTER) { { - State = 911; + State = 969; Match(OUTER); } } @@ -5930,14 +6229,14 @@ public JoinTypeContext joinType() { _localctx = new FullJoinContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 914; + State = 972; Match(FULL); - State = 916; + State = 974; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==OUTER) { { - State = 915; + State = 973; Match(OUTER); } } @@ -5984,9 +6283,9 @@ public WhereClauseContext whereClause() { try { EnterOuterAlt(_localctx, 1); { - State = 920; + State = 978; Match(WHERE); - State = 921; + State = 979; expression(0); } } @@ -6035,25 +6334,25 @@ public OrderByClauseContext orderByClause() { try { EnterOuterAlt(_localctx, 1); { - State = 923; + State = 981; Match(ORDER); - State = 924; + State = 982; Match(BY); - State = 925; + State = 983; orderByItem(); - State = 930; + State = 988; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 926; + State = 984; Match(COMMA); - State = 927; + State = 985; orderByItem(); } } - State = 932; + State = 990; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -6098,14 +6397,14 @@ public OrderByItemContext orderByItem() { try { EnterOuterAlt(_localctx, 1); { - State = 933; + State = 991; expression(0); - State = 935; + State = 993; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ASC || _la==DESC) { { - State = 934; + State = 992; _localctx.dir = TokenStream.LT(1); _la = TokenStream.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -6145,14 +6444,16 @@ public virtual void CopyFrom(ExpressionContext context) { } public partial class AndExprContext : ExpressionContext { public ExpressionContext left; + public IToken op; public ExpressionContext right; - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode AND() { return GetToken(AccessSqlParser.AND, 0); } [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { return GetRuleContexts(); } [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { return GetRuleContext(i); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode AND() { return GetToken(AccessSqlParser.AND, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BAND() { return GetToken(AccessSqlParser.BAND, 0); } public AndExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { @@ -6161,6 +6462,25 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } + public partial class IntDivExprContext : ExpressionContext { + public ExpressionContext left; + public IToken op; + public ExpressionContext right; + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BACKSLASH() { return GetToken(AccessSqlParser.BACKSLASH, 0); } + public IntDivExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitIntDivExpr(this); + else return visitor.VisitChildren(this); + } + } public partial class LikeExprContext : ExpressionContext { public ExpressionContext left; public IToken not; @@ -6222,6 +6542,26 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } + public partial class XorExprContext : ExpressionContext { + public ExpressionContext left; + public IToken op; + public ExpressionContext right; + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode XOR() { return GetToken(AccessSqlParser.XOR, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BXOR() { return GetToken(AccessSqlParser.BXOR, 0); } + public XorExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitXorExpr(this); + else return visitor.VisitChildren(this); + } + } public partial class BetweenExprContext : ExpressionContext { public ExpressionContext val; public IToken not; @@ -6244,19 +6584,6 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } - public partial class BitNotExprContext : ExpressionContext { - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BNOT() { return GetToken(AccessSqlParser.BNOT, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression() { - return GetRuleContext(0); - } - public BitNotExprContext(ExpressionContext context) { CopyFrom(context); } - [System.Diagnostics.DebuggerNonUserCode] - public override TResult Accept(IParseTreeVisitor visitor) { - IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; - if (typedVisitor != null) return typedVisitor.VisitBitNotExpr(this); - else return visitor.VisitChildren(this); - } - } public partial class PowExprContext : ExpressionContext { public ExpressionContext left; public ExpressionContext right; @@ -6277,14 +6604,16 @@ public override TResult Accept(IParseTreeVisitor visitor) { } public partial class OrExprContext : ExpressionContext { public ExpressionContext left; + public IToken op; public ExpressionContext right; - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode OR() { return GetToken(AccessSqlParser.OR, 0); } [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { return GetRuleContexts(); } [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { return GetRuleContext(i); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode OR() { return GetToken(AccessSqlParser.OR, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BOR() { return GetToken(AccessSqlParser.BOR, 0); } public OrExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { @@ -6294,10 +6623,12 @@ public override TResult Accept(IParseTreeVisitor visitor) { } } public partial class NegateExprContext : ExpressionContext { - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode MINUS() { return GetToken(AccessSqlParser.MINUS, 0); } + public IToken op; [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression() { return GetRuleContext(0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode MINUS() { return GetToken(AccessSqlParser.MINUS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode PLUS() { return GetToken(AccessSqlParser.PLUS, 0); } public NegateExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { @@ -6328,7 +6659,7 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } - public partial class MulDivExprContext : ExpressionContext { + public partial class ConcatExprContext : ExpressionContext { public ExpressionContext left; public IToken op; public ExpressionContext right; @@ -6338,19 +6669,16 @@ [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { return GetRuleContext(i); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode STAR() { return GetToken(AccessSqlParser.STAR, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode SLASH() { return GetToken(AccessSqlParser.SLASH, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode MOD() { return GetToken(AccessSqlParser.MOD, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BACKSLASH() { return GetToken(AccessSqlParser.BACKSLASH, 0); } - public MulDivExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode AMP() { return GetToken(AccessSqlParser.AMP, 0); } + public ConcatExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; - if (typedVisitor != null) return typedVisitor.VisitMulDivExpr(this); + if (typedVisitor != null) return typedVisitor.VisitConcatExpr(this); else return visitor.VisitChildren(this); } } - public partial class BitwiseExprContext : ExpressionContext { + public partial class MulDivExprContext : ExpressionContext { public ExpressionContext left; public IToken op; public ExpressionContext right; @@ -6360,18 +6688,36 @@ [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { return GetRuleContext(i); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BAND() { return GetToken(AccessSqlParser.BAND, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BOR() { return GetToken(AccessSqlParser.BOR, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BXOR() { return GetToken(AccessSqlParser.BXOR, 0); } - public BitwiseExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode STAR() { return GetToken(AccessSqlParser.STAR, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode SLASH() { return GetToken(AccessSqlParser.SLASH, 0); } + public MulDivExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; - if (typedVisitor != null) return typedVisitor.VisitBitwiseExpr(this); + if (typedVisitor != null) return typedVisitor.VisitMulDivExpr(this); else return visitor.VisitChildren(this); } } - public partial class PrimaryExprContext : ExpressionContext { + public partial class ImpExprContext : ExpressionContext { + public ExpressionContext left; + public IToken op; + public ExpressionContext right; + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode IMP() { return GetToken(AccessSqlParser.IMP, 0); } + public ImpExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitImpExpr(this); + else return visitor.VisitChildren(this); + } + } + public partial class PrimaryExprContext : ExpressionContext { [System.Diagnostics.DebuggerNonUserCode] public PrimaryContext primary() { return GetRuleContext(0); } @@ -6384,10 +6730,12 @@ public override TResult Accept(IParseTreeVisitor visitor) { } } public partial class NotExprContext : ExpressionContext { - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NOT() { return GetToken(AccessSqlParser.NOT, 0); } + public IToken op; [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression() { return GetRuleContext(0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NOT() { return GetToken(AccessSqlParser.NOT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BNOT() { return GetToken(AccessSqlParser.BNOT, 0); } public NotExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { @@ -6396,6 +6744,25 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } + public partial class ModExprContext : ExpressionContext { + public ExpressionContext left; + public IToken op; + public ExpressionContext right; + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode MOD() { return GetToken(AccessSqlParser.MOD, 0); } + public ModExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitModExpr(this); + else return visitor.VisitChildren(this); + } + } public partial class InExprContext : ExpressionContext { public ExpressionContext val; public IToken not; @@ -6423,7 +6790,7 @@ public override TResult Accept(IParseTreeVisitor visitor) { else return visitor.VisitChildren(this); } } - public partial class AddConcatExprContext : ExpressionContext { + public partial class AddSubExprContext : ExpressionContext { public ExpressionContext left; public IToken op; public ExpressionContext right; @@ -6435,12 +6802,30 @@ [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode PLUS() { return GetToken(AccessSqlParser.PLUS, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode MINUS() { return GetToken(AccessSqlParser.MINUS, 0); } - [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode AMP() { return GetToken(AccessSqlParser.AMP, 0); } - public AddConcatExprContext(ExpressionContext context) { CopyFrom(context); } + public AddSubExprContext(ExpressionContext context) { CopyFrom(context); } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitAddSubExpr(this); + else return visitor.VisitChildren(this); + } + } + public partial class EqvExprContext : ExpressionContext { + public ExpressionContext left; + public IToken op; + public ExpressionContext right; + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode EQV() { return GetToken(AccessSqlParser.EQV, 0); } + public EqvExprContext(ExpressionContext context) { CopyFrom(context); } [System.Diagnostics.DebuggerNonUserCode] public override TResult Accept(IParseTreeVisitor visitor) { IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; - if (typedVisitor != null) return typedVisitor.VisitAddConcatExpr(this); + if (typedVisitor != null) return typedVisitor.VisitEqvExpr(this); else return visitor.VisitChildren(this); } } @@ -6462,41 +6847,48 @@ private ExpressionContext expression(int _p) { int _alt; EnterOuterAlt(_localctx, 1); { - State = 945; + State = 1001; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { - case NOT: + case PLUS: + case MINUS: { - _localctx = new NotExprContext(_localctx); + _localctx = new NegateExprContext(_localctx); Context = _localctx; _prevctx = _localctx; - State = 938; - Match(NOT); - State = 939; - expression(16); + State = 996; + ((NegateExprContext)_localctx).op = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==PLUS || _la==MINUS) ) { + ((NegateExprContext)_localctx).op = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 997; + expression(19); } break; + case NOT: case BNOT: { - _localctx = new BitNotExprContext(_localctx); + _localctx = new NotExprContext(_localctx); Context = _localctx; _prevctx = _localctx; - State = 940; - Match(BNOT); - State = 941; - expression(15); + State = 998; + ((NotExprContext)_localctx).op = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==NOT || _la==BNOT) ) { + ((NotExprContext)_localctx).op = ErrorHandler.RecoverInline(this); } - break; - case MINUS: - { - _localctx = new NegateExprContext(_localctx); - Context = _localctx; - _prevctx = _localctx; - State = 942; - Match(MINUS); - State = 943; - expression(14); + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 999; + expression(7); } break; case LEFT: @@ -6509,6 +6901,20 @@ private ExpressionContext expression(int _p) { case TRUE: case FALSE: case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case LPAREN: case SYSVAR: case PARAM: @@ -6525,7 +6931,7 @@ private ExpressionContext expression(int _p) { _localctx = new PrimaryExprContext(_localctx); Context = _localctx; _prevctx = _localctx; - State = 944; + State = 1000; primary(); } break; @@ -6533,29 +6939,29 @@ private ExpressionContext expression(int _p) { throw new NoViableAltException(this); } Context.Stop = TokenStream.LT(-1); - State = 1016; + State = 1087; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,124,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,133,Context); while ( _alt!=2 && _alt!=global::Antlr4.Runtime.Atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( ParseListeners!=null ) TriggerExitRuleEvent(); _prevctx = _localctx; { - State = 1014; + State = 1085; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,123,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,132,Context) ) { case 1: { _localctx = new PowExprContext(new ExpressionContext(_parentctx, _parentState)); ((PowExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 947; - if (!(Precpred(Context, 13))) throw new FailedPredicateException(this, "Precpred(Context, 13)"); - State = 948; + State = 1003; + if (!(Precpred(Context, 20))) throw new FailedPredicateException(this, "Precpred(Context, 20)"); + State = 1004; Match(CARET); - State = 949; - ((PowExprContext)_localctx).right = expression(14); + State = 1005; + ((PowExprContext)_localctx).right = expression(21); } break; case 2: @@ -6563,262 +6969,343 @@ private ExpressionContext expression(int _p) { _localctx = new MulDivExprContext(new ExpressionContext(_parentctx, _parentState)); ((MulDivExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 950; - if (!(Precpred(Context, 12))) throw new FailedPredicateException(this, "Precpred(Context, 12)"); - State = 951; + State = 1006; + if (!(Precpred(Context, 18))) throw new FailedPredicateException(this, "Precpred(Context, 18)"); + State = 1007; ((MulDivExprContext)_localctx).op = TokenStream.LT(1); _la = TokenStream.LA(1); - if ( !(_la==MOD || ((((_la - 102)) & ~0x3f) == 0 && ((1L << (_la - 102)) & 7L) != 0)) ) { + if ( !(_la==STAR || _la==SLASH) ) { ((MulDivExprContext)_localctx).op = ErrorHandler.RecoverInline(this); } else { ErrorHandler.ReportMatch(this); Consume(); } - State = 952; - ((MulDivExprContext)_localctx).right = expression(13); + State = 1008; + ((MulDivExprContext)_localctx).right = expression(19); } break; case 3: { - _localctx = new AddConcatExprContext(new ExpressionContext(_parentctx, _parentState)); - ((AddConcatExprContext)_localctx).left = _prevctx; + _localctx = new IntDivExprContext(new ExpressionContext(_parentctx, _parentState)); + ((IntDivExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 953; - if (!(Precpred(Context, 11))) throw new FailedPredicateException(this, "Precpred(Context, 11)"); - State = 954; - ((AddConcatExprContext)_localctx).op = TokenStream.LT(1); + State = 1009; + if (!(Precpred(Context, 17))) throw new FailedPredicateException(this, "Precpred(Context, 17)"); + State = 1010; + ((IntDivExprContext)_localctx).op = Match(BACKSLASH); + State = 1011; + ((IntDivExprContext)_localctx).right = expression(18); + } + break; + case 4: + { + _localctx = new ModExprContext(new ExpressionContext(_parentctx, _parentState)); + ((ModExprContext)_localctx).left = _prevctx; + PushNewRecursionContext(_localctx, _startState, RULE_expression); + State = 1012; + if (!(Precpred(Context, 16))) throw new FailedPredicateException(this, "Precpred(Context, 16)"); + State = 1013; + ((ModExprContext)_localctx).op = Match(MOD); + State = 1014; + ((ModExprContext)_localctx).right = expression(17); + } + break; + case 5: + { + _localctx = new AddSubExprContext(new ExpressionContext(_parentctx, _parentState)); + ((AddSubExprContext)_localctx).left = _prevctx; + PushNewRecursionContext(_localctx, _startState, RULE_expression); + State = 1015; + if (!(Precpred(Context, 15))) throw new FailedPredicateException(this, "Precpred(Context, 15)"); + State = 1016; + ((AddSubExprContext)_localctx).op = TokenStream.LT(1); _la = TokenStream.LA(1); - if ( !(((((_la - 106)) & ~0x3f) == 0 && ((1L << (_la - 106)) & 7L) != 0)) ) { - ((AddConcatExprContext)_localctx).op = ErrorHandler.RecoverInline(this); + if ( !(_la==PLUS || _la==MINUS) ) { + ((AddSubExprContext)_localctx).op = ErrorHandler.RecoverInline(this); } else { ErrorHandler.ReportMatch(this); Consume(); } - State = 955; - ((AddConcatExprContext)_localctx).right = expression(12); + State = 1017; + ((AddSubExprContext)_localctx).right = expression(16); } break; - case 4: + case 6: + { + _localctx = new ConcatExprContext(new ExpressionContext(_parentctx, _parentState)); + ((ConcatExprContext)_localctx).left = _prevctx; + PushNewRecursionContext(_localctx, _startState, RULE_expression); + State = 1018; + if (!(Precpred(Context, 14))) throw new FailedPredicateException(this, "Precpred(Context, 14)"); + State = 1019; + ((ConcatExprContext)_localctx).op = Match(AMP); + State = 1020; + ((ConcatExprContext)_localctx).right = expression(15); + } + break; + case 7: { _localctx = new ComparisonExprContext(new ExpressionContext(_parentctx, _parentState)); ((ComparisonExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 956; - if (!(Precpred(Context, 10))) throw new FailedPredicateException(this, "Precpred(Context, 10)"); - State = 957; + State = 1021; + if (!(Precpred(Context, 13))) throw new FailedPredicateException(this, "Precpred(Context, 13)"); + State = 1022; ((ComparisonExprContext)_localctx).op = TokenStream.LT(1); _la = TokenStream.LA(1); - if ( !(((((_la - 109)) & ~0x3f) == 0 && ((1L << (_la - 109)) & 63L) != 0)) ) { + if ( !(((((_la - 129)) & ~0x3f) == 0 && ((1L << (_la - 129)) & 63L) != 0)) ) { ((ComparisonExprContext)_localctx).op = ErrorHandler.RecoverInline(this); } else { ErrorHandler.ReportMatch(this); Consume(); } - State = 958; - ((ComparisonExprContext)_localctx).right = expression(11); + State = 1023; + ((ComparisonExprContext)_localctx).right = expression(14); } break; - case 5: + case 8: { _localctx = new BetweenExprContext(new ExpressionContext(_parentctx, _parentState)); ((BetweenExprContext)_localctx).val = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 959; - if (!(Precpred(Context, 9))) throw new FailedPredicateException(this, "Precpred(Context, 9)"); - State = 961; + State = 1024; + if (!(Precpred(Context, 12))) throw new FailedPredicateException(this, "Precpred(Context, 12)"); + State = 1026; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 960; + State = 1025; ((BetweenExprContext)_localctx).not = Match(NOT); } } - State = 963; + State = 1028; Match(BETWEEN); - State = 964; + State = 1029; ((BetweenExprContext)_localctx).lo = expression(0); - State = 965; + State = 1030; Match(AND); - State = 966; - ((BetweenExprContext)_localctx).hi = expression(10); + State = 1031; + ((BetweenExprContext)_localctx).hi = expression(13); } break; - case 6: + case 9: { _localctx = new LikeExprContext(new ExpressionContext(_parentctx, _parentState)); ((LikeExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 968; - if (!(Precpred(Context, 8))) throw new FailedPredicateException(this, "Precpred(Context, 8)"); - State = 970; + State = 1033; + if (!(Precpred(Context, 11))) throw new FailedPredicateException(this, "Precpred(Context, 11)"); + State = 1035; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 969; + State = 1034; ((LikeExprContext)_localctx).not = Match(NOT); } } - State = 972; + State = 1037; Match(LIKE); - State = 973; - ((LikeExprContext)_localctx).right = expression(9); + State = 1038; + ((LikeExprContext)_localctx).right = expression(12); } break; - case 7: + case 10: + { + _localctx = new AndExprContext(new ExpressionContext(_parentctx, _parentState)); + ((AndExprContext)_localctx).left = _prevctx; + PushNewRecursionContext(_localctx, _startState, RULE_expression); + State = 1039; + if (!(Precpred(Context, 6))) throw new FailedPredicateException(this, "Precpred(Context, 6)"); + State = 1040; + ((AndExprContext)_localctx).op = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==AND || _la==BAND) ) { + ((AndExprContext)_localctx).op = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 1041; + ((AndExprContext)_localctx).right = expression(7); + } + break; + case 11: + { + _localctx = new OrExprContext(new ExpressionContext(_parentctx, _parentState)); + ((OrExprContext)_localctx).left = _prevctx; + PushNewRecursionContext(_localctx, _startState, RULE_expression); + State = 1042; + if (!(Precpred(Context, 5))) throw new FailedPredicateException(this, "Precpred(Context, 5)"); + State = 1043; + ((OrExprContext)_localctx).op = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==OR || _la==BOR) ) { + ((OrExprContext)_localctx).op = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 1044; + ((OrExprContext)_localctx).right = expression(6); + } + break; + case 12: { - _localctx = new BitwiseExprContext(new ExpressionContext(_parentctx, _parentState)); - ((BitwiseExprContext)_localctx).left = _prevctx; + _localctx = new XorExprContext(new ExpressionContext(_parentctx, _parentState)); + ((XorExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 974; + State = 1045; if (!(Precpred(Context, 4))) throw new FailedPredicateException(this, "Precpred(Context, 4)"); - State = 975; - ((BitwiseExprContext)_localctx).op = TokenStream.LT(1); + State = 1046; + ((XorExprContext)_localctx).op = TokenStream.LT(1); _la = TokenStream.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 3584L) != 0)) ) { - ((BitwiseExprContext)_localctx).op = ErrorHandler.RecoverInline(this); + if ( !(_la==XOR || _la==BXOR) ) { + ((XorExprContext)_localctx).op = ErrorHandler.RecoverInline(this); } else { ErrorHandler.ReportMatch(this); Consume(); } - State = 976; - ((BitwiseExprContext)_localctx).right = expression(5); + State = 1047; + ((XorExprContext)_localctx).right = expression(5); } break; - case 8: + case 13: { - _localctx = new AndExprContext(new ExpressionContext(_parentctx, _parentState)); - ((AndExprContext)_localctx).left = _prevctx; + _localctx = new EqvExprContext(new ExpressionContext(_parentctx, _parentState)); + ((EqvExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 977; + State = 1048; if (!(Precpred(Context, 3))) throw new FailedPredicateException(this, "Precpred(Context, 3)"); - State = 978; - Match(AND); - State = 979; - ((AndExprContext)_localctx).right = expression(4); + State = 1049; + ((EqvExprContext)_localctx).op = Match(EQV); + State = 1050; + ((EqvExprContext)_localctx).right = expression(4); } break; - case 9: + case 14: { - _localctx = new OrExprContext(new ExpressionContext(_parentctx, _parentState)); - ((OrExprContext)_localctx).left = _prevctx; + _localctx = new ImpExprContext(new ExpressionContext(_parentctx, _parentState)); + ((ImpExprContext)_localctx).left = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 980; + State = 1051; if (!(Precpred(Context, 2))) throw new FailedPredicateException(this, "Precpred(Context, 2)"); - State = 981; - Match(OR); - State = 982; - ((OrExprContext)_localctx).right = expression(3); + State = 1052; + ((ImpExprContext)_localctx).op = Match(IMP); + State = 1053; + ((ImpExprContext)_localctx).right = expression(3); } break; - case 10: + case 15: { _localctx = new InSubqueryExprContext(new ExpressionContext(_parentctx, _parentState)); ((InSubqueryExprContext)_localctx).val = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 983; - if (!(Precpred(Context, 7))) throw new FailedPredicateException(this, "Precpred(Context, 7)"); - State = 985; + State = 1054; + if (!(Precpred(Context, 10))) throw new FailedPredicateException(this, "Precpred(Context, 10)"); + State = 1056; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 984; + State = 1055; ((InSubqueryExprContext)_localctx).not = Match(NOT); } } - State = 987; + State = 1058; Match(IN); - State = 988; + State = 1059; Match(LPAREN); - State = 989; + State = 1060; ((InSubqueryExprContext)_localctx).sub = queryExpression(); - State = 990; + State = 1061; Match(RPAREN); } break; - case 11: + case 16: { _localctx = new InExprContext(new ExpressionContext(_parentctx, _parentState)); ((InExprContext)_localctx).val = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 992; - if (!(Precpred(Context, 6))) throw new FailedPredicateException(this, "Precpred(Context, 6)"); - State = 994; + State = 1063; + if (!(Precpred(Context, 9))) throw new FailedPredicateException(this, "Precpred(Context, 9)"); + State = 1065; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 993; + State = 1064; ((InExprContext)_localctx).not = Match(NOT); } } - State = 996; + State = 1067; Match(IN); - State = 997; + State = 1068; Match(LPAREN); - State = 998; + State = 1069; ((InExprContext)_localctx)._expression = expression(0); ((InExprContext)_localctx)._items.Add(((InExprContext)_localctx)._expression); - State = 1003; + State = 1074; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 999; + State = 1070; Match(COMMA); - State = 1000; + State = 1071; ((InExprContext)_localctx)._expression = expression(0); ((InExprContext)_localctx)._items.Add(((InExprContext)_localctx)._expression); } } - State = 1005; + State = 1076; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } - State = 1006; + State = 1077; Match(RPAREN); } break; - case 12: + case 17: { _localctx = new IsNullExprContext(new ExpressionContext(_parentctx, _parentState)); ((IsNullExprContext)_localctx).operand = _prevctx; PushNewRecursionContext(_localctx, _startState, RULE_expression); - State = 1008; - if (!(Precpred(Context, 5))) throw new FailedPredicateException(this, "Precpred(Context, 5)"); - State = 1009; + State = 1079; + if (!(Precpred(Context, 8))) throw new FailedPredicateException(this, "Precpred(Context, 8)"); + State = 1080; Match(IS); - State = 1011; + State = 1082; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==NOT) { { - State = 1010; + State = 1081; ((IsNullExprContext)_localctx).not = Match(NOT); } } - State = 1013; + State = 1084; Match(NULL); } break; } } } - State = 1018; + State = 1089; ErrorHandler.Sync(this); - _alt = Interpreter.AdaptivePredict(TokenStream,124,Context); + _alt = Interpreter.AdaptivePredict(TokenStream,133,Context); } } } @@ -6962,14 +7449,14 @@ public PrimaryContext primary() { PrimaryContext _localctx = new PrimaryContext(Context, State); EnterRule(_localctx, 114, RULE_primary); try { - State = 1038; + State = 1109; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,125,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,134,Context) ) { case 1: _localctx = new LiteralPrimaryContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 1019; + State = 1090; literal(); } break; @@ -6977,7 +7464,7 @@ public PrimaryContext primary() { _localctx = new CasePrimaryContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 1020; + State = 1091; caseExpression(); } break; @@ -6985,7 +7472,7 @@ public PrimaryContext primary() { _localctx = new FunctionCallPrimaryContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 1021; + State = 1092; functionCall(); } break; @@ -6993,7 +7480,7 @@ public PrimaryContext primary() { _localctx = new ColumnPrimaryContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 1022; + State = 1093; columnRef(); } break; @@ -7001,7 +7488,7 @@ public PrimaryContext primary() { _localctx = new ParamPrimaryContext(_localctx); EnterOuterAlt(_localctx, 5); { - State = 1023; + State = 1094; Match(PARAM); } break; @@ -7009,7 +7496,7 @@ public PrimaryContext primary() { _localctx = new SystemVariablePrimaryContext(_localctx); EnterOuterAlt(_localctx, 6); { - State = 1024; + State = 1095; Match(SYSVAR); } break; @@ -7017,13 +7504,13 @@ public PrimaryContext primary() { _localctx = new ExistsPrimaryContext(_localctx); EnterOuterAlt(_localctx, 7); { - State = 1025; + State = 1096; Match(EXISTS); - State = 1026; + State = 1097; Match(LPAREN); - State = 1027; + State = 1098; queryExpression(); - State = 1028; + State = 1099; Match(RPAREN); } break; @@ -7031,11 +7518,11 @@ public PrimaryContext primary() { _localctx = new ScalarSubqueryPrimaryContext(_localctx); EnterOuterAlt(_localctx, 8); { - State = 1030; + State = 1101; Match(LPAREN); - State = 1031; + State = 1102; queryExpression(); - State = 1032; + State = 1103; Match(RPAREN); } break; @@ -7043,11 +7530,11 @@ public PrimaryContext primary() { _localctx = new ParenPrimaryContext(_localctx); EnterOuterAlt(_localctx, 9); { - State = 1034; + State = 1105; Match(LPAREN); - State = 1035; + State = 1106; expression(0); - State = 1036; + State = 1107; Match(RPAREN); } break; @@ -7103,45 +7590,45 @@ public CaseExpressionContext caseExpression() { try { EnterOuterAlt(_localctx, 1); { - State = 1040; + State = 1111; Match(CASE); - State = 1042; + State = 1113; ErrorHandler.Sync(this); _la = TokenStream.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 35596957585664L) != 0) || ((((_la - 97)) & ~0x3f) == 0 && ((1L << (_la - 97)) & 17171743773L) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 284775660683520L) != 0) || ((((_la - 103)) & ~0x3f) == 0 && ((1L << (_la - 103)) & 281341858414589L) != 0)) { { - State = 1041; + State = 1112; _localctx.operand = expression(0); } } - State = 1045; + State = 1116; ErrorHandler.Sync(this); _la = TokenStream.LA(1); do { { { - State = 1044; + State = 1115; caseWhen(); } } - State = 1047; + State = 1118; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } while ( _la==WHEN ); - State = 1051; + State = 1122; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ELSE) { { - State = 1049; + State = 1120; Match(ELSE); - State = 1050; + State = 1121; _localctx.elseResult = expression(0); } } - State = 1053; + State = 1124; Match(END); } } @@ -7187,13 +7674,13 @@ public CaseWhenContext caseWhen() { try { EnterOuterAlt(_localctx, 1); { - State = 1055; + State = 1126; Match(WHEN); - State = 1056; + State = 1127; _localctx.condition = expression(0); - State = 1057; + State = 1128; Match(THEN); - State = 1058; + State = 1129; _localctx.result = expression(0); } } @@ -7217,6 +7704,12 @@ public partial class FunctionCallContext : ParserRuleContext { [System.Diagnostics.DebuggerNonUserCode] public FunctionNameContext functionName() { return GetRuleContext(0); } + [System.Diagnostics.DebuggerNonUserCode] public WithinGroupContext withinGroup() { + return GetRuleContext(0); + } + [System.Diagnostics.DebuggerNonUserCode] public FilterClauseContext filterClause() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode OVER() { return GetToken(AccessSqlParser.OVER, 0); } [System.Diagnostics.DebuggerNonUserCode] public WindowSpecificationContext windowSpecification() { return GetRuleContext(0); @@ -7228,6 +7721,12 @@ [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression(int i) { return GetRuleContext(i); } + [System.Diagnostics.DebuggerNonUserCode] public NthRowFromContext nthRowFrom() { + return GetRuleContext(0); + } + [System.Diagnostics.DebuggerNonUserCode] public NullTreatmentContext nullTreatment() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode[] COMMA() { return GetTokens(AccessSqlParser.COMMA); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode COMMA(int i) { return GetToken(AccessSqlParser.COMMA, i); @@ -7254,16 +7753,16 @@ public FunctionCallContext functionCall() { try { EnterOuterAlt(_localctx, 1); { - State = 1060; + State = 1131; _localctx.name = functionName(); - State = 1061; + State = 1132; Match(LPAREN); - State = 1074; + State = 1145; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case STAR: { - State = 1062; + State = 1133; _localctx.star = Match(STAR); } break; @@ -7280,6 +7779,21 @@ public FunctionCallContext functionCall() { case TRUE: case FALSE: case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + case PLUS: case MINUS: case LPAREN: case SYSVAR: @@ -7295,31 +7809,31 @@ public FunctionCallContext functionCall() { case IDENTIFIER: { { - State = 1064; + State = 1135; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==DISTINCT) { { - State = 1063; + State = 1134; _localctx.distinct = Match(DISTINCT); } } - State = 1066; + State = 1137; expression(0); - State = 1071; + State = 1142; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 1067; + State = 1138; Match(COMMA); - State = 1068; + State = 1139; expression(0); } } - State = 1073; + State = 1144; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } @@ -7331,16 +7845,56 @@ public FunctionCallContext functionCall() { default: break; } - State = 1076; + State = 1147; Match(RPAREN); - State = 1079; + State = 1149; + ErrorHandler.Sync(this); + switch ( Interpreter.AdaptivePredict(TokenStream,141,Context) ) { + case 1: + { + State = 1148; + withinGroup(); + } + break; + } + State = 1152; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,132,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,142,Context) ) { case 1: { - State = 1077; + State = 1151; + filterClause(); + } + break; + } + State = 1162; + ErrorHandler.Sync(this); + switch ( Interpreter.AdaptivePredict(TokenStream,145,Context) ) { + case 1: + { + State = 1155; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==FROM) { + { + State = 1154; + nthRowFrom(); + } + } + + State = 1158; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==IGNORE || _la==RESPECT) { + { + State = 1157; + nullTreatment(); + } + } + + State = 1160; Match(OVER); - State = 1078; + State = 1161; windowSpecification(); } break; @@ -7385,50 +7939,64 @@ public FunctionNameContext functionName() { FunctionNameContext _localctx = new FunctionNameContext(Context, State); EnterRule(_localctx, 122, RULE_functionName); try { - State = 1087; + State = 1170; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: case BRACKET_ID: case BACKTICK_ID: case IDENTIFIER: EnterOuterAlt(_localctx, 1); { - State = 1081; + State = 1164; identifier(); } break; case LEFT: EnterOuterAlt(_localctx, 2); { - State = 1082; + State = 1165; Match(LEFT); } break; case RIGHT: EnterOuterAlt(_localctx, 3); { - State = 1083; + State = 1166; Match(RIGHT); } break; case ASC: EnterOuterAlt(_localctx, 4); { - State = 1084; + State = 1167; Match(ASC); } break; case FIRST: EnterOuterAlt(_localctx, 5); { - State = 1085; + State = 1168; Match(FIRST); } break; case PARTITION: EnterOuterAlt(_localctx, 6); { - State = 1086; + State = 1169; Match(PARTITION); } break; @@ -7477,19 +8045,19 @@ public ColumnRefContext columnRef() { try { EnterOuterAlt(_localctx, 1); { - State = 1092; + State = 1175; ErrorHandler.Sync(this); - switch ( Interpreter.AdaptivePredict(TokenStream,134,Context) ) { + switch ( Interpreter.AdaptivePredict(TokenStream,147,Context) ) { case 1: { - State = 1089; + State = 1172; _localctx.qualifier = identifier(); - State = 1090; + State = 1173; Match(DOT); } break; } - State = 1094; + State = 1177; _localctx.name = identifier(); } } @@ -7508,6 +8076,9 @@ public partial class IdentifierContext : ParserRuleContext { [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode IDENTIFIER() { return GetToken(AccessSqlParser.IDENTIFIER, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BRACKET_ID() { return GetToken(AccessSqlParser.BRACKET_ID, 0); } [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BACKTICK_ID() { return GetToken(AccessSqlParser.BACKTICK_ID, 0); } + [System.Diagnostics.DebuggerNonUserCode] public NonReservedKeywordContext nonReservedKeyword() { + return GetRuleContext(0); + } public IdentifierContext(ParserRuleContext parent, int invokingState) : base(parent, invokingState) { @@ -7525,19 +8096,53 @@ public override TResult Accept(IParseTreeVisitor visitor) { public IdentifierContext identifier() { IdentifierContext _localctx = new IdentifierContext(Context, State); EnterRule(_localctx, 126, RULE_identifier); - int _la; try { - EnterOuterAlt(_localctx, 1); - { - State = 1096; - _la = TokenStream.LA(1); - if ( !(((((_la - 128)) & ~0x3f) == 0 && ((1L << (_la - 128)) & 7L) != 0)) ) { - ErrorHandler.RecoverInline(this); - } - else { - ErrorHandler.ReportMatch(this); - Consume(); - } + State = 1183; + ErrorHandler.Sync(this); + switch (TokenStream.LA(1)) { + case IDENTIFIER: + EnterOuterAlt(_localctx, 1); + { + State = 1179; + Match(IDENTIFIER); + } + break; + case BRACKET_ID: + EnterOuterAlt(_localctx, 2); + { + State = 1180; + Match(BRACKET_ID); + } + break; + case BACKTICK_ID: + EnterOuterAlt(_localctx, 3); + { + State = 1181; + Match(BACKTICK_ID); + } + break; + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + EnterOuterAlt(_localctx, 4); + { + State = 1182; + nonReservedKeyword(); + } + break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -7659,14 +8264,14 @@ public LiteralContext literal() { LiteralContext _localctx = new LiteralContext(Context, State); EnterRule(_localctx, 128, RULE_literal); try { - State = 1107; + State = 1194; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case INTEGER_LITERAL: _localctx = new IntLiteralContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 1098; + State = 1185; Match(INTEGER_LITERAL); } break; @@ -7674,7 +8279,7 @@ public LiteralContext literal() { _localctx = new NumberLiteralContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 1099; + State = 1186; Match(NUMBER_LITERAL); } break; @@ -7682,7 +8287,7 @@ public LiteralContext literal() { _localctx = new HexLiteralContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 1100; + State = 1187; Match(HEX_LITERAL); } break; @@ -7690,7 +8295,7 @@ public LiteralContext literal() { _localctx = new StringLiteralContext(_localctx); EnterOuterAlt(_localctx, 4); { - State = 1101; + State = 1188; Match(STRING_LITERAL); } break; @@ -7698,7 +8303,7 @@ public LiteralContext literal() { _localctx = new DateLiteralContext(_localctx); EnterOuterAlt(_localctx, 5); { - State = 1102; + State = 1189; Match(DATE_LITERAL); } break; @@ -7706,7 +8311,7 @@ public LiteralContext literal() { _localctx = new GuidLiteralContext(_localctx); EnterOuterAlt(_localctx, 6); { - State = 1103; + State = 1190; Match(GUID_LITERAL); } break; @@ -7714,7 +8319,7 @@ public LiteralContext literal() { _localctx = new TrueLiteralContext(_localctx); EnterOuterAlt(_localctx, 7); { - State = 1104; + State = 1191; Match(TRUE); } break; @@ -7722,7 +8327,7 @@ public LiteralContext literal() { _localctx = new FalseLiteralContext(_localctx); EnterOuterAlt(_localctx, 8); { - State = 1105; + State = 1192; Match(FALSE); } break; @@ -7730,7 +8335,7 @@ public LiteralContext literal() { _localctx = new NullLiteralContext(_localctx); EnterOuterAlt(_localctx, 9); { - State = 1106; + State = 1193; Match(NULL); } break; @@ -7804,21 +8409,21 @@ public TransactionStatementContext transactionStatement() { EnterRule(_localctx, 130, RULE_transactionStatement); int _la; try { - State = 1121; + State = 1208; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { case BEGIN: _localctx = new BeginTransactionStatementContext(_localctx); EnterOuterAlt(_localctx, 1); { - State = 1109; + State = 1196; Match(BEGIN); - State = 1111; + State = 1198; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==TRANSACTION || _la==WORK) { { - State = 1110; + State = 1197; _la = TokenStream.LA(1); if ( !(_la==TRANSACTION || _la==WORK) ) { ErrorHandler.RecoverInline(this); @@ -7836,14 +8441,14 @@ public TransactionStatementContext transactionStatement() { _localctx = new CommitTransactionStatementContext(_localctx); EnterOuterAlt(_localctx, 2); { - State = 1113; + State = 1200; Match(COMMIT); - State = 1115; + State = 1202; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==TRANSACTION || _la==WORK) { { - State = 1114; + State = 1201; _la = TokenStream.LA(1); if ( !(_la==TRANSACTION || _la==WORK) ) { ErrorHandler.RecoverInline(this); @@ -7861,14 +8466,14 @@ public TransactionStatementContext transactionStatement() { _localctx = new RollbackTransactionStatementContext(_localctx); EnterOuterAlt(_localctx, 3); { - State = 1117; + State = 1204; Match(ROLLBACK); - State = 1119; + State = 1206; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==TRANSACTION || _la==WORK) { { - State = 1118; + State = 1205; _la = TokenStream.LA(1); if ( !(_la==TRANSACTION || _la==WORK) ) { ErrorHandler.RecoverInline(this); @@ -7922,9 +8527,9 @@ public StandaloneExpressionContext standaloneExpression() { try { EnterOuterAlt(_localctx, 1); { - State = 1123; + State = 1210; expression(0); - State = 1124; + State = 1211; Match(Eof); } } @@ -7949,6 +8554,9 @@ public partial class WindowSpecificationContext : ParserRuleContext { [System.Diagnostics.DebuggerNonUserCode] public OrderByClauseContext orderByClause() { return GetRuleContext(0); } + [System.Diagnostics.DebuggerNonUserCode] public WindowFrameContext windowFrame() { + return GetRuleContext(0); + } [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext[] expression() { return GetRuleContexts(); } @@ -7980,51 +8588,520 @@ public WindowSpecificationContext windowSpecification() { try { EnterOuterAlt(_localctx, 1); { - State = 1126; + State = 1213; Match(LPAREN); - State = 1137; + State = 1224; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==PARTITION) { { - State = 1127; + State = 1214; Match(PARTITION); - State = 1128; + State = 1215; Match(BY); - State = 1129; + State = 1216; _localctx._expression = expression(0); _localctx._partition.Add(_localctx._expression); - State = 1134; + State = 1221; ErrorHandler.Sync(this); _la = TokenStream.LA(1); while (_la==COMMA) { { { - State = 1130; + State = 1217; Match(COMMA); - State = 1131; + State = 1218; _localctx._expression = expression(0); _localctx._partition.Add(_localctx._expression); } } - State = 1136; + State = 1223; ErrorHandler.Sync(this); _la = TokenStream.LA(1); } } } - State = 1140; + State = 1227; ErrorHandler.Sync(this); _la = TokenStream.LA(1); if (_la==ORDER) { { - State = 1139; + State = 1226; orderByClause(); } } - State = 1142; + State = 1230; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (((((_la - 49)) & ~0x3f) == 0 && ((1L << (_la - 49)) & 1729382256910270465L) != 0)) { + { + State = 1229; + windowFrame(); + } + } + + State = 1232; + Match(RPAREN); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class ClusteredOptionContext : ParserRuleContext { + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CLUSTERED() { return GetToken(AccessSqlParser.CLUSTERED, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NONCLUSTERED() { return GetToken(AccessSqlParser.NONCLUSTERED, 0); } + public ClusteredOptionContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_clusteredOption; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitClusteredOption(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public ClusteredOptionContext clusteredOption() { + ClusteredOptionContext _localctx = new ClusteredOptionContext(Context, State); + EnterRule(_localctx, 136, RULE_clusteredOption); + int _la; + try { + EnterOuterAlt(_localctx, 1); + { + State = 1234; + _la = TokenStream.LA(1); + if ( !(_la==CLUSTERED || _la==NONCLUSTERED) ) { + ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class WindowFrameContext : ParserRuleContext { + public IToken unit; + public FrameBoundContext start; + public FrameBoundContext end; + public FrameExclusionContext exclusion; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode ROWS() { return GetToken(AccessSqlParser.ROWS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RANGE() { return GetToken(AccessSqlParser.RANGE, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode GROUPS() { return GetToken(AccessSqlParser.GROUPS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode BETWEEN() { return GetToken(AccessSqlParser.BETWEEN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode AND() { return GetToken(AccessSqlParser.AND, 0); } + [System.Diagnostics.DebuggerNonUserCode] public FrameBoundContext[] frameBound() { + return GetRuleContexts(); + } + [System.Diagnostics.DebuggerNonUserCode] public FrameBoundContext frameBound(int i) { + return GetRuleContext(i); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode EXCLUDE() { return GetToken(AccessSqlParser.EXCLUDE, 0); } + [System.Diagnostics.DebuggerNonUserCode] public FrameExclusionContext frameExclusion() { + return GetRuleContext(0); + } + public WindowFrameContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_windowFrame; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitWindowFrame(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public WindowFrameContext windowFrame() { + WindowFrameContext _localctx = new WindowFrameContext(Context, State); + EnterRule(_localctx, 138, RULE_windowFrame); + int _la; + try { + EnterOuterAlt(_localctx, 1); + { + State = 1236; + _localctx.unit = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(((((_la - 49)) & ~0x3f) == 0 && ((1L << (_la - 49)) & 1729382256910270465L) != 0)) ) { + _localctx.unit = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 1243; + ErrorHandler.Sync(this); + switch (TokenStream.LA(1)) { + case BETWEEN: + { + State = 1237; + Match(BETWEEN); + State = 1238; + _localctx.start = frameBound(); + State = 1239; + Match(AND); + State = 1240; + _localctx.end = frameBound(); + } + break; + case NOT: + case BNOT: + case LEFT: + case RIGHT: + case EXISTS: + case PARTITION: + case CASE: + case FIRST: + case ASC: + case TRUE: + case FALSE: + case NULL: + case RANGE: + case GROUPS: + case UNBOUNDED: + case PRECEDING: + case FOLLOWING: + case CURRENT: + case EXCLUDE: + case TIES: + case OTHERS: + case WITHIN: + case LAST: + case RESPECT: + case NULLS: + case FILTER: + case PLUS: + case MINUS: + case LPAREN: + case SYSVAR: + case PARAM: + case HEX_LITERAL: + case INTEGER_LITERAL: + case NUMBER_LITERAL: + case STRING_LITERAL: + case DATE_LITERAL: + case GUID_LITERAL: + case BRACKET_ID: + case BACKTICK_ID: + case IDENTIFIER: + { + State = 1242; + _localctx.start = frameBound(); + } + break; + default: + throw new NoViableAltException(this); + } + State = 1247; + ErrorHandler.Sync(this); + _la = TokenStream.LA(1); + if (_la==EXCLUDE) { + { + State = 1245; + Match(EXCLUDE); + State = 1246; + _localctx.exclusion = frameExclusion(); + } + } + + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class FrameBoundContext : ParserRuleContext { + public IToken direction; + public ExpressionContext offset; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode UNBOUNDED() { return GetToken(AccessSqlParser.UNBOUNDED, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode PRECEDING() { return GetToken(AccessSqlParser.PRECEDING, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FOLLOWING() { return GetToken(AccessSqlParser.FOLLOWING, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CURRENT() { return GetToken(AccessSqlParser.CURRENT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode ROW() { return GetToken(AccessSqlParser.ROW, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression() { + return GetRuleContext(0); + } + public FrameBoundContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_frameBound; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitFrameBound(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public FrameBoundContext frameBound() { + FrameBoundContext _localctx = new FrameBoundContext(Context, State); + EnterRule(_localctx, 140, RULE_frameBound); + int _la; + try { + State = 1256; + ErrorHandler.Sync(this); + switch ( Interpreter.AdaptivePredict(TokenStream,160,Context) ) { + case 1: + EnterOuterAlt(_localctx, 1); + { + State = 1249; + Match(UNBOUNDED); + State = 1250; + _localctx.direction = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==PRECEDING || _la==FOLLOWING) ) { + _localctx.direction = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + } + break; + case 2: + EnterOuterAlt(_localctx, 2); + { + State = 1251; + Match(CURRENT); + State = 1252; + Match(ROW); + } + break; + case 3: + EnterOuterAlt(_localctx, 3); + { + State = 1253; + _localctx.offset = expression(0); + State = 1254; + _localctx.direction = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==PRECEDING || _la==FOLLOWING) ) { + _localctx.direction = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class FrameExclusionContext : ParserRuleContext { + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CURRENT() { return GetToken(AccessSqlParser.CURRENT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode ROW() { return GetToken(AccessSqlParser.ROW, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode GROUP() { return GetToken(AccessSqlParser.GROUP, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode TIES() { return GetToken(AccessSqlParser.TIES, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NO() { return GetToken(AccessSqlParser.NO, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode OTHERS() { return GetToken(AccessSqlParser.OTHERS, 0); } + public FrameExclusionContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_frameExclusion; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitFrameExclusion(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public FrameExclusionContext frameExclusion() { + FrameExclusionContext _localctx = new FrameExclusionContext(Context, State); + EnterRule(_localctx, 142, RULE_frameExclusion); + try { + State = 1264; + ErrorHandler.Sync(this); + switch (TokenStream.LA(1)) { + case CURRENT: + EnterOuterAlt(_localctx, 1); + { + State = 1258; + Match(CURRENT); + State = 1259; + Match(ROW); + } + break; + case GROUP: + EnterOuterAlt(_localctx, 2); + { + State = 1260; + Match(GROUP); + } + break; + case TIES: + EnterOuterAlt(_localctx, 3); + { + State = 1261; + Match(TIES); + } + break; + case NO: + EnterOuterAlt(_localctx, 4); + { + State = 1262; + Match(NO); + State = 1263; + Match(OTHERS); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class NonReservedKeywordContext : ParserRuleContext { + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RANGE() { return GetToken(AccessSqlParser.RANGE, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode GROUPS() { return GetToken(AccessSqlParser.GROUPS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode UNBOUNDED() { return GetToken(AccessSqlParser.UNBOUNDED, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode PRECEDING() { return GetToken(AccessSqlParser.PRECEDING, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FOLLOWING() { return GetToken(AccessSqlParser.FOLLOWING, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode CURRENT() { return GetToken(AccessSqlParser.CURRENT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode EXCLUDE() { return GetToken(AccessSqlParser.EXCLUDE, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode TIES() { return GetToken(AccessSqlParser.TIES, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode OTHERS() { return GetToken(AccessSqlParser.OTHERS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode WITHIN() { return GetToken(AccessSqlParser.WITHIN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LAST() { return GetToken(AccessSqlParser.LAST, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RESPECT() { return GetToken(AccessSqlParser.RESPECT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NULLS() { return GetToken(AccessSqlParser.NULLS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FILTER() { return GetToken(AccessSqlParser.FILTER, 0); } + public NonReservedKeywordContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_nonReservedKeyword; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitNonReservedKeyword(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public NonReservedKeywordContext nonReservedKeyword() { + NonReservedKeywordContext _localctx = new NonReservedKeywordContext(Context, State); + EnterRule(_localctx, 144, RULE_nonReservedKeyword); + int _la; + try { + EnterOuterAlt(_localctx, 1); + { + State = 1266; + _la = TokenStream.LA(1); + if ( !(((((_la - 108)) & ~0x3f) == 0 && ((1L << (_la - 108)) & 16383L) != 0)) ) { + ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class FilterClauseContext : ParserRuleContext { + public ExpressionContext condition; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FILTER() { return GetToken(AccessSqlParser.FILTER, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LPAREN() { return GetToken(AccessSqlParser.LPAREN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode WHERE() { return GetToken(AccessSqlParser.WHERE, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RPAREN() { return GetToken(AccessSqlParser.RPAREN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ExpressionContext expression() { + return GetRuleContext(0); + } + public FilterClauseContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_filterClause; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitFilterClause(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public FilterClauseContext filterClause() { + FilterClauseContext _localctx = new FilterClauseContext(Context, State); + EnterRule(_localctx, 146, RULE_filterClause); + try { + EnterOuterAlt(_localctx, 1); + { + State = 1268; + Match(FILTER); + State = 1269; + Match(LPAREN); + State = 1270; + Match(WHERE); + State = 1271; + _localctx.condition = expression(0); + State = 1272; Match(RPAREN); } } @@ -8039,6 +9116,159 @@ public WindowSpecificationContext windowSpecification() { return _localctx; } + public partial class WithinGroupContext : ParserRuleContext { + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode WITHIN() { return GetToken(AccessSqlParser.WITHIN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode GROUP() { return GetToken(AccessSqlParser.GROUP, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LPAREN() { return GetToken(AccessSqlParser.LPAREN, 0); } + [System.Diagnostics.DebuggerNonUserCode] public OrderByClauseContext orderByClause() { + return GetRuleContext(0); + } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RPAREN() { return GetToken(AccessSqlParser.RPAREN, 0); } + public WithinGroupContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_withinGroup; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitWithinGroup(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public WithinGroupContext withinGroup() { + WithinGroupContext _localctx = new WithinGroupContext(Context, State); + EnterRule(_localctx, 148, RULE_withinGroup); + try { + EnterOuterAlt(_localctx, 1); + { + State = 1274; + Match(WITHIN); + State = 1275; + Match(GROUP); + State = 1276; + Match(LPAREN); + State = 1277; + orderByClause(); + State = 1278; + Match(RPAREN); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class NthRowFromContext : ParserRuleContext { + public IToken edge; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FROM() { return GetToken(AccessSqlParser.FROM, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode FIRST() { return GetToken(AccessSqlParser.FIRST, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode LAST() { return GetToken(AccessSqlParser.LAST, 0); } + public NthRowFromContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_nthRowFrom; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitNthRowFrom(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public NthRowFromContext nthRowFrom() { + NthRowFromContext _localctx = new NthRowFromContext(Context, State); + EnterRule(_localctx, 150, RULE_nthRowFrom); + int _la; + try { + EnterOuterAlt(_localctx, 1); + { + State = 1280; + Match(FROM); + State = 1281; + _localctx.edge = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==FIRST || _la==LAST) ) { + _localctx.edge = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + + public partial class NullTreatmentContext : ParserRuleContext { + public IToken treatment; + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode NULLS() { return GetToken(AccessSqlParser.NULLS, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode RESPECT() { return GetToken(AccessSqlParser.RESPECT, 0); } + [System.Diagnostics.DebuggerNonUserCode] public ITerminalNode IGNORE() { return GetToken(AccessSqlParser.IGNORE, 0); } + public NullTreatmentContext(ParserRuleContext parent, int invokingState) + : base(parent, invokingState) + { + } + public override int RuleIndex { get { return RULE_nullTreatment; } } + [System.Diagnostics.DebuggerNonUserCode] + public override TResult Accept(IParseTreeVisitor visitor) { + IAccessSqlVisitor typedVisitor = visitor as IAccessSqlVisitor; + if (typedVisitor != null) return typedVisitor.VisitNullTreatment(this); + else return visitor.VisitChildren(this); + } + } + + [RuleVersion(0)] + public NullTreatmentContext nullTreatment() { + NullTreatmentContext _localctx = new NullTreatmentContext(Context, State); + EnterRule(_localctx, 152, RULE_nullTreatment); + int _la; + try { + EnterOuterAlt(_localctx, 1); + { + State = 1283; + _localctx.treatment = TokenStream.LT(1); + _la = TokenStream.LA(1); + if ( !(_la==IGNORE || _la==RESPECT) ) { + _localctx.treatment = ErrorHandler.RecoverInline(this); + } + else { + ErrorHandler.ReportMatch(this); + Consume(); + } + State = 1284; + Match(NULLS); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + ErrorHandler.ReportError(this, re); + ErrorHandler.Recover(this, re); + } + finally { + ExitRule(); + } + return _localctx; + } + public override bool Sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { case 56: return expression_sempred((ExpressionContext)_localctx, predIndex); @@ -8047,24 +9277,29 @@ public override bool Sempred(RuleContext _localctx, int ruleIndex, int predIndex } private bool expression_sempred(ExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 0: return Precpred(Context, 13); - case 1: return Precpred(Context, 12); - case 2: return Precpred(Context, 11); - case 3: return Precpred(Context, 10); - case 4: return Precpred(Context, 9); - case 5: return Precpred(Context, 8); - case 6: return Precpred(Context, 4); - case 7: return Precpred(Context, 3); - case 8: return Precpred(Context, 2); - case 9: return Precpred(Context, 7); - case 10: return Precpred(Context, 6); - case 11: return Precpred(Context, 5); + case 0: return Precpred(Context, 20); + case 1: return Precpred(Context, 18); + case 2: return Precpred(Context, 17); + case 3: return Precpred(Context, 16); + case 4: return Precpred(Context, 15); + case 5: return Precpred(Context, 14); + case 6: return Precpred(Context, 13); + case 7: return Precpred(Context, 12); + case 8: return Precpred(Context, 11); + case 9: return Precpred(Context, 6); + case 10: return Precpred(Context, 5); + case 11: return Precpred(Context, 4); + case 12: return Precpred(Context, 3); + case 13: return Precpred(Context, 2); + case 14: return Precpred(Context, 10); + case 15: return Precpred(Context, 9); + case 16: return Precpred(Context, 8); } return true; } private static int[] _serializedATN = { - 4,1,133,1145,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,6,2, + 4,1,153,1287,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,6,2, 7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,2,11,7,11,2,12,7,12,2,13,7,13,2,14,7,14, 2,15,7,15,2,16,7,16,2,17,7,17,2,18,7,18,2,19,7,19,2,20,7,20,2,21,7,21, 2,22,7,22,2,23,7,23,2,24,7,24,2,25,7,25,2,26,7,26,2,27,7,27,2,28,7,28, @@ -8073,413 +9308,471 @@ private bool expression_sempred(ExpressionContext _localctx, int predIndex) { 2,43,7,43,2,44,7,44,2,45,7,45,2,46,7,46,2,47,7,47,2,48,7,48,2,49,7,49, 2,50,7,50,2,51,7,51,2,52,7,52,2,53,7,53,2,54,7,54,2,55,7,55,2,56,7,56, 2,57,7,57,2,58,7,58,2,59,7,59,2,60,7,60,2,61,7,61,2,62,7,62,2,63,7,63, - 2,64,7,64,2,65,7,65,2,66,7,66,2,67,7,67,1,0,3,0,138,8,0,1,0,1,0,1,0,1, - 0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,3,0,154,8,0,1,0,3,0,157,8,0, - 1,0,1,0,1,1,1,1,3,1,163,8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,2,1,2,1, - 2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,3,2,183,8,2,1,3,1,3,1,3,1,3,1,3,5,3,190, - 8,3,10,3,12,3,193,9,3,3,3,195,8,3,1,4,1,4,1,4,1,4,1,4,1,4,5,4,203,8,4, - 10,4,12,4,206,9,4,1,4,3,4,209,8,4,1,5,1,5,1,5,1,5,1,6,1,6,1,6,1,6,1,6, - 1,6,3,6,221,8,6,1,6,1,6,1,6,3,6,226,8,6,1,7,1,7,1,7,1,7,5,7,232,8,7,10, - 7,12,7,235,9,7,1,8,1,8,3,8,239,8,8,1,8,3,8,242,8,8,1,9,1,9,1,9,1,9,5,9, - 248,8,9,10,9,12,9,251,9,9,1,9,1,9,1,10,1,10,3,10,257,8,10,1,10,1,10,1, - 10,1,10,1,10,1,10,5,10,265,8,10,10,10,12,10,268,9,10,1,10,1,10,5,10,272, - 8,10,10,10,12,10,275,9,10,1,10,1,10,1,11,1,11,1,11,1,11,1,11,1,11,1,11, - 5,11,286,8,11,10,11,12,11,289,9,11,1,11,1,11,3,11,293,8,11,1,11,1,11,1, - 11,1,12,1,12,1,12,1,12,3,12,302,8,12,1,12,1,12,1,12,1,13,1,13,1,13,1,13, - 5,13,311,8,13,10,13,12,13,314,9,13,1,13,1,13,1,13,1,13,1,13,5,13,321,8, - 13,10,13,12,13,324,9,13,3,13,326,8,13,1,14,1,14,1,14,1,15,1,15,3,15,333, - 8,15,1,16,1,16,1,16,1,16,1,16,1,17,1,17,3,17,342,8,17,1,17,1,17,1,17,1, - 17,1,17,3,17,349,8,17,1,17,1,17,1,17,5,17,354,8,17,10,17,12,17,357,9,17, - 1,17,1,17,3,17,361,8,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,3,17,370,8, - 17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1, - 17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,3,17,397,8,17, - 1,18,1,18,1,18,3,18,402,8,18,1,19,1,19,3,19,406,8,19,1,19,1,19,1,19,1, - 19,1,19,1,19,1,19,1,19,5,19,416,8,19,10,19,12,19,419,9,19,1,19,1,19,1, - 19,3,19,424,8,19,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20, - 1,20,1,20,1,20,1,20,3,20,441,8,20,1,21,1,21,3,21,445,8,21,1,22,1,22,1, - 22,1,22,1,22,3,22,452,8,22,1,23,1,23,1,23,3,23,457,8,23,1,23,5,23,460, - 8,23,10,23,12,23,463,9,23,1,24,1,24,1,24,1,24,1,24,1,25,1,25,3,25,472, - 8,25,1,25,3,25,475,8,25,1,25,1,25,1,25,1,25,3,25,481,8,25,1,25,1,25,3, - 25,485,8,25,1,26,3,26,488,8,26,1,26,1,26,1,27,1,27,1,27,1,27,1,27,1,27, - 1,27,1,27,1,27,3,27,501,8,27,1,27,1,27,1,27,1,27,1,27,1,27,1,27,3,27,510, - 8,27,1,27,1,27,1,27,1,27,3,27,516,8,27,1,27,1,27,1,27,3,27,521,8,27,1, - 27,1,27,1,27,1,27,1,27,1,27,5,27,529,8,27,10,27,12,27,532,9,27,1,27,1, - 27,3,27,536,8,27,1,27,5,27,539,8,27,10,27,12,27,542,9,27,3,27,544,8,27, - 1,28,1,28,3,28,548,8,28,1,28,1,28,1,28,1,28,1,28,1,28,5,28,556,8,28,10, - 28,12,28,559,9,28,1,28,1,28,1,28,1,28,3,28,565,8,28,1,28,1,28,1,28,1,28, - 1,28,5,28,572,8,28,10,28,12,28,575,9,28,1,28,1,28,1,28,1,28,3,28,581,8, - 28,1,28,1,28,1,28,1,28,3,28,587,8,28,1,28,1,28,1,28,1,28,5,28,593,8,28, - 10,28,12,28,596,9,28,1,28,1,28,1,28,1,28,1,28,1,28,1,28,5,28,605,8,28, - 10,28,12,28,608,9,28,1,28,1,28,3,28,612,8,28,1,28,5,28,615,8,28,10,28, - 12,28,618,9,28,1,28,1,28,3,28,622,8,28,1,28,1,28,1,28,1,28,1,28,3,28,629, - 8,28,1,29,1,29,1,29,1,29,1,29,5,29,636,8,29,10,29,12,29,639,9,29,1,30, - 1,30,1,30,1,30,1,30,1,30,3,30,647,8,30,1,31,1,31,1,31,1,31,1,31,1,31,1, - 31,1,31,3,31,657,8,31,1,32,1,32,1,32,1,32,1,32,1,32,1,32,5,32,666,8,32, - 10,32,12,32,669,9,32,1,32,1,32,3,32,673,8,32,1,32,1,32,1,32,1,32,5,32, - 679,8,32,10,32,12,32,682,9,32,1,32,3,32,685,8,32,1,32,1,32,3,32,689,8, - 32,1,33,1,33,1,33,1,33,5,33,695,8,33,10,33,12,33,698,9,33,1,33,1,33,1, - 34,1,34,3,34,704,8,34,1,35,1,35,1,35,1,35,5,35,710,8,35,10,35,12,35,713, - 9,35,1,35,3,35,716,8,35,1,35,3,35,719,8,35,1,36,1,36,1,36,1,36,1,36,1, - 36,1,36,1,36,1,36,5,36,730,8,36,10,36,12,36,733,9,36,3,36,735,8,36,1,37, - 1,37,3,37,739,8,37,1,37,1,37,3,37,743,8,37,1,38,1,38,3,38,747,8,38,1,38, - 3,38,750,8,38,1,38,1,38,1,38,3,38,755,8,38,1,38,3,38,758,8,38,1,38,3,38, - 761,8,38,1,38,3,38,764,8,38,1,38,3,38,767,8,38,1,39,1,39,1,40,1,40,1,40, - 1,40,1,40,5,40,776,8,40,10,40,12,40,779,9,40,1,41,1,41,1,41,1,42,1,42, - 1,42,1,42,5,42,788,8,42,10,42,12,42,791,9,42,1,42,3,42,794,8,42,1,43,1, - 43,1,43,1,43,1,43,1,43,3,43,802,8,43,1,44,1,44,1,44,1,44,1,44,1,44,1,44, - 1,44,1,44,3,44,813,8,44,1,44,1,44,1,44,1,44,1,44,1,44,3,44,821,8,44,1, - 45,1,45,1,46,1,46,1,46,1,46,5,46,829,8,46,10,46,12,46,832,9,46,3,46,834, - 8,46,1,47,1,47,1,47,1,47,1,47,1,47,3,47,842,8,47,1,47,3,47,845,8,47,3, - 47,847,8,47,1,48,1,48,1,48,1,48,5,48,853,8,48,10,48,12,48,856,9,48,1,49, - 1,49,5,49,860,8,49,10,49,12,49,863,9,49,1,50,1,50,3,50,867,8,50,1,50,3, - 50,870,8,50,1,50,1,50,1,50,1,50,3,50,876,8,50,1,50,3,50,879,8,50,1,50, - 1,50,1,50,1,50,3,50,885,8,50,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1, - 51,1,51,1,51,1,51,1,51,1,51,1,51,3,51,902,8,51,1,52,3,52,905,8,52,1,52, - 1,52,3,52,909,8,52,1,52,1,52,3,52,913,8,52,1,52,1,52,3,52,917,8,52,3,52, - 919,8,52,1,53,1,53,1,53,1,54,1,54,1,54,1,54,1,54,5,54,929,8,54,10,54,12, - 54,932,9,54,1,55,1,55,3,55,936,8,55,1,56,1,56,1,56,1,56,1,56,1,56,1,56, - 1,56,3,56,946,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1, - 56,1,56,1,56,1,56,3,56,962,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,3,56, - 971,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1, - 56,3,56,986,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,3,56,995,8,56,1,56, - 1,56,1,56,1,56,1,56,5,56,1002,8,56,10,56,12,56,1005,9,56,1,56,1,56,1,56, - 1,56,1,56,3,56,1012,8,56,1,56,5,56,1015,8,56,10,56,12,56,1018,9,56,1,57, - 1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57, - 1,57,1,57,1,57,1,57,3,57,1039,8,57,1,58,1,58,3,58,1043,8,58,1,58,4,58, - 1046,8,58,11,58,12,58,1047,1,58,1,58,3,58,1052,8,58,1,58,1,58,1,59,1,59, - 1,59,1,59,1,59,1,60,1,60,1,60,1,60,3,60,1065,8,60,1,60,1,60,1,60,5,60, - 1070,8,60,10,60,12,60,1073,9,60,3,60,1075,8,60,1,60,1,60,1,60,3,60,1080, - 8,60,1,61,1,61,1,61,1,61,1,61,1,61,3,61,1088,8,61,1,62,1,62,1,62,3,62, - 1093,8,62,1,62,1,62,1,63,1,63,1,64,1,64,1,64,1,64,1,64,1,64,1,64,1,64, - 1,64,3,64,1108,8,64,1,65,1,65,3,65,1112,8,65,1,65,1,65,3,65,1116,8,65, - 1,65,1,65,3,65,1120,8,65,3,65,1122,8,65,1,66,1,66,1,66,1,67,1,67,1,67, - 1,67,1,67,1,67,5,67,1133,8,67,10,67,12,67,1136,9,67,3,67,1138,8,67,1,67, - 3,67,1141,8,67,1,67,1,67,1,67,0,1,112,68,0,2,4,6,8,10,12,14,16,18,20,22, - 24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70, - 72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112, - 114,116,118,120,122,124,126,128,130,132,134,0,14,1,0,95,96,1,0,97,98,1, - 0,87,88,1,0,115,116,2,0,31,32,51,51,1,0,106,107,1,0,44,45,1,0,46,47,2, - 0,14,14,102,104,1,0,106,108,1,0,109,114,1,0,9,11,1,0,128,130,1,0,59,60, - 1304,0,137,1,0,0,0,2,160,1,0,0,0,4,182,1,0,0,0,6,184,1,0,0,0,8,196,1,0, - 0,0,10,210,1,0,0,0,12,214,1,0,0,0,14,227,1,0,0,0,16,236,1,0,0,0,18,243, - 1,0,0,0,20,254,1,0,0,0,22,278,1,0,0,0,24,297,1,0,0,0,26,325,1,0,0,0,28, - 327,1,0,0,0,30,332,1,0,0,0,32,334,1,0,0,0,34,396,1,0,0,0,36,401,1,0,0, - 0,38,403,1,0,0,0,40,440,1,0,0,0,42,442,1,0,0,0,44,451,1,0,0,0,46,453,1, - 0,0,0,48,464,1,0,0,0,50,469,1,0,0,0,52,487,1,0,0,0,54,543,1,0,0,0,56,628, - 1,0,0,0,58,637,1,0,0,0,60,646,1,0,0,0,62,656,1,0,0,0,64,658,1,0,0,0,66, - 690,1,0,0,0,68,703,1,0,0,0,70,705,1,0,0,0,72,734,1,0,0,0,74,742,1,0,0, - 0,76,744,1,0,0,0,78,768,1,0,0,0,80,770,1,0,0,0,82,780,1,0,0,0,84,783,1, - 0,0,0,86,801,1,0,0,0,88,820,1,0,0,0,90,822,1,0,0,0,92,833,1,0,0,0,94,846, - 1,0,0,0,96,848,1,0,0,0,98,857,1,0,0,0,100,884,1,0,0,0,102,901,1,0,0,0, - 104,918,1,0,0,0,106,920,1,0,0,0,108,923,1,0,0,0,110,933,1,0,0,0,112,945, - 1,0,0,0,114,1038,1,0,0,0,116,1040,1,0,0,0,118,1055,1,0,0,0,120,1060,1, - 0,0,0,122,1087,1,0,0,0,124,1092,1,0,0,0,126,1096,1,0,0,0,128,1107,1,0, - 0,0,130,1121,1,0,0,0,132,1123,1,0,0,0,134,1126,1,0,0,0,136,138,3,18,9, - 0,137,136,1,0,0,0,137,138,1,0,0,0,138,153,1,0,0,0,139,154,3,2,1,0,140, - 154,3,20,10,0,141,154,3,38,19,0,142,154,3,22,11,0,143,154,3,24,12,0,144, - 154,3,32,16,0,145,154,3,40,20,0,146,154,3,64,32,0,147,154,3,8,4,0,148, - 154,3,12,6,0,149,154,3,130,65,0,150,154,3,6,3,0,151,154,3,14,7,0,152,154, - 3,70,35,0,153,139,1,0,0,0,153,140,1,0,0,0,153,141,1,0,0,0,153,142,1,0, - 0,0,153,143,1,0,0,0,153,144,1,0,0,0,153,145,1,0,0,0,153,146,1,0,0,0,153, - 147,1,0,0,0,153,148,1,0,0,0,153,149,1,0,0,0,153,150,1,0,0,0,153,151,1, - 0,0,0,153,152,1,0,0,0,154,156,1,0,0,0,155,157,5,119,0,0,156,155,1,0,0, - 0,156,157,1,0,0,0,157,158,1,0,0,0,158,159,5,0,0,1,159,1,1,0,0,0,160,162, - 5,29,0,0,161,163,5,8,0,0,162,161,1,0,0,0,162,163,1,0,0,0,163,164,1,0,0, - 0,164,165,5,28,0,0,165,166,5,115,0,0,166,167,3,70,35,0,167,168,5,116,0, - 0,168,169,5,30,0,0,169,170,3,4,2,0,170,3,1,0,0,0,171,183,3,20,10,0,172, - 183,3,38,19,0,173,183,3,22,11,0,174,183,3,24,12,0,175,183,3,32,16,0,176, - 183,3,40,20,0,177,183,3,64,32,0,178,183,3,8,4,0,179,183,3,12,6,0,180,183, - 3,6,3,0,181,183,3,70,35,0,182,171,1,0,0,0,182,172,1,0,0,0,182,173,1,0, - 0,0,182,174,1,0,0,0,182,175,1,0,0,0,182,176,1,0,0,0,182,177,1,0,0,0,182, - 178,1,0,0,0,182,179,1,0,0,0,182,180,1,0,0,0,182,181,1,0,0,0,183,5,1,0, - 0,0,184,185,7,0,0,0,185,194,3,126,63,0,186,191,3,112,56,0,187,188,5,117, - 0,0,188,190,3,112,56,0,189,187,1,0,0,0,190,193,1,0,0,0,191,189,1,0,0,0, - 191,192,1,0,0,0,192,195,1,0,0,0,193,191,1,0,0,0,194,186,1,0,0,0,194,195, - 1,0,0,0,195,7,1,0,0,0,196,197,5,76,0,0,197,198,3,98,49,0,198,199,5,80, - 0,0,199,204,3,10,5,0,200,201,5,117,0,0,201,203,3,10,5,0,202,200,1,0,0, - 0,203,206,1,0,0,0,204,202,1,0,0,0,204,205,1,0,0,0,205,208,1,0,0,0,206, - 204,1,0,0,0,207,209,3,106,53,0,208,207,1,0,0,0,208,209,1,0,0,0,209,9,1, - 0,0,0,210,211,3,124,62,0,211,212,5,109,0,0,212,213,3,112,56,0,213,11,1, - 0,0,0,214,220,5,75,0,0,215,216,3,126,63,0,216,217,5,118,0,0,217,218,5, - 102,0,0,218,221,1,0,0,0,219,221,5,102,0,0,220,215,1,0,0,0,220,219,1,0, - 0,0,220,221,1,0,0,0,221,222,1,0,0,0,222,223,5,2,0,0,223,225,3,98,49,0, - 224,226,3,106,53,0,225,224,1,0,0,0,225,226,1,0,0,0,226,13,1,0,0,0,227, - 228,5,1,0,0,228,233,3,16,8,0,229,230,5,117,0,0,230,232,3,16,8,0,231,229, - 1,0,0,0,232,235,1,0,0,0,233,231,1,0,0,0,233,234,1,0,0,0,234,15,1,0,0,0, - 235,233,1,0,0,0,236,241,5,120,0,0,237,239,5,5,0,0,238,237,1,0,0,0,238, - 239,1,0,0,0,239,240,1,0,0,0,240,242,3,126,63,0,241,238,1,0,0,0,241,242, - 1,0,0,0,242,17,1,0,0,0,243,244,5,94,0,0,244,249,3,28,14,0,245,246,5,117, - 0,0,246,248,3,28,14,0,247,245,1,0,0,0,248,251,1,0,0,0,249,247,1,0,0,0, - 249,250,1,0,0,0,250,252,1,0,0,0,251,249,1,0,0,0,252,253,5,119,0,0,253, - 19,1,0,0,0,254,256,5,54,0,0,255,257,5,85,0,0,256,255,1,0,0,0,256,257,1, - 0,0,0,257,258,1,0,0,0,258,259,5,55,0,0,259,260,3,126,63,0,260,261,5,115, - 0,0,261,266,3,46,23,0,262,263,5,117,0,0,263,265,3,46,23,0,264,262,1,0, - 0,0,265,268,1,0,0,0,266,264,1,0,0,0,266,267,1,0,0,0,267,273,1,0,0,0,268, - 266,1,0,0,0,269,270,5,117,0,0,270,272,3,56,28,0,271,269,1,0,0,0,272,275, - 1,0,0,0,273,271,1,0,0,0,273,274,1,0,0,0,274,276,1,0,0,0,275,273,1,0,0, - 0,276,277,5,116,0,0,277,21,1,0,0,0,278,279,5,54,0,0,279,280,5,92,0,0,280, - 292,3,126,63,0,281,282,5,115,0,0,282,287,3,126,63,0,283,284,5,117,0,0, - 284,286,3,126,63,0,285,283,1,0,0,0,286,289,1,0,0,0,287,285,1,0,0,0,287, - 288,1,0,0,0,288,290,1,0,0,0,289,287,1,0,0,0,290,291,5,116,0,0,291,293, - 1,0,0,0,292,281,1,0,0,0,292,293,1,0,0,0,293,294,1,0,0,0,294,295,5,5,0, - 0,295,296,3,70,35,0,296,23,1,0,0,0,297,298,5,54,0,0,298,299,5,93,0,0,299, - 301,3,126,63,0,300,302,3,26,13,0,301,300,1,0,0,0,301,302,1,0,0,0,302,303, - 1,0,0,0,303,304,5,5,0,0,304,305,3,36,18,0,305,25,1,0,0,0,306,307,5,115, - 0,0,307,312,3,28,14,0,308,309,5,117,0,0,309,311,3,28,14,0,310,308,1,0, - 0,0,311,314,1,0,0,0,312,310,1,0,0,0,312,313,1,0,0,0,313,315,1,0,0,0,314, - 312,1,0,0,0,315,316,5,116,0,0,316,326,1,0,0,0,317,322,3,28,14,0,318,319, - 5,117,0,0,319,321,3,28,14,0,320,318,1,0,0,0,321,324,1,0,0,0,322,320,1, - 0,0,0,322,323,1,0,0,0,323,326,1,0,0,0,324,322,1,0,0,0,325,306,1,0,0,0, - 325,317,1,0,0,0,326,27,1,0,0,0,327,328,3,30,15,0,328,329,3,50,25,0,329, - 29,1,0,0,0,330,333,3,126,63,0,331,333,5,121,0,0,332,330,1,0,0,0,332,331, - 1,0,0,0,333,31,1,0,0,0,334,335,5,61,0,0,335,336,5,55,0,0,336,337,3,126, - 63,0,337,338,3,34,17,0,338,33,1,0,0,0,339,341,5,64,0,0,340,342,5,66,0, - 0,341,340,1,0,0,0,341,342,1,0,0,0,342,343,1,0,0,0,343,397,3,46,23,0,344, - 345,5,64,0,0,345,397,3,56,28,0,346,348,5,61,0,0,347,349,5,66,0,0,348,347, - 1,0,0,0,348,349,1,0,0,0,349,350,1,0,0,0,350,351,3,126,63,0,351,355,3,50, - 25,0,352,354,3,54,27,0,353,352,1,0,0,0,354,357,1,0,0,0,355,353,1,0,0,0, - 355,356,1,0,0,0,356,397,1,0,0,0,357,355,1,0,0,0,358,360,5,61,0,0,359,361, - 5,66,0,0,360,359,1,0,0,0,360,361,1,0,0,0,361,362,1,0,0,0,362,363,3,126, - 63,0,363,364,5,80,0,0,364,365,5,81,0,0,365,366,3,112,56,0,366,397,1,0, - 0,0,367,369,5,61,0,0,368,370,5,66,0,0,369,368,1,0,0,0,369,370,1,0,0,0, - 370,371,1,0,0,0,371,372,3,126,63,0,372,373,5,65,0,0,373,374,5,81,0,0,374, - 397,1,0,0,0,375,376,5,65,0,0,376,377,5,66,0,0,377,397,3,126,63,0,378,379, - 5,65,0,0,379,380,5,72,0,0,380,397,3,126,63,0,381,382,5,62,0,0,382,383, - 5,63,0,0,383,397,3,126,63,0,384,385,5,62,0,0,385,386,5,66,0,0,386,387, - 3,126,63,0,387,388,5,63,0,0,388,389,3,126,63,0,389,397,1,0,0,0,390,391, - 5,62,0,0,391,392,5,84,0,0,392,393,3,126,63,0,393,394,5,63,0,0,394,395, - 3,126,63,0,395,397,1,0,0,0,396,339,1,0,0,0,396,344,1,0,0,0,396,346,1,0, - 0,0,396,358,1,0,0,0,396,367,1,0,0,0,396,375,1,0,0,0,396,378,1,0,0,0,396, - 381,1,0,0,0,396,384,1,0,0,0,396,390,1,0,0,0,397,35,1,0,0,0,398,402,3,70, - 35,0,399,402,3,64,32,0,400,402,3,20,10,0,401,398,1,0,0,0,401,399,1,0,0, - 0,401,400,1,0,0,0,402,37,1,0,0,0,403,405,5,54,0,0,404,406,5,83,0,0,405, - 404,1,0,0,0,405,406,1,0,0,0,406,407,1,0,0,0,407,408,5,84,0,0,408,409,3, - 126,63,0,409,410,5,22,0,0,410,411,3,126,63,0,411,412,5,115,0,0,412,417, - 3,42,21,0,413,414,5,117,0,0,414,416,3,42,21,0,415,413,1,0,0,0,416,419, - 1,0,0,0,417,415,1,0,0,0,417,418,1,0,0,0,418,420,1,0,0,0,419,417,1,0,0, - 0,420,423,5,116,0,0,421,422,5,86,0,0,422,424,3,44,22,0,423,421,1,0,0,0, - 423,424,1,0,0,0,424,39,1,0,0,0,425,426,5,65,0,0,426,427,5,55,0,0,427,441, - 3,126,63,0,428,429,5,65,0,0,429,430,5,84,0,0,430,431,3,126,63,0,431,432, - 5,22,0,0,432,433,3,126,63,0,433,441,1,0,0,0,434,435,5,65,0,0,435,436,5, - 93,0,0,436,441,3,126,63,0,437,438,5,65,0,0,438,439,5,92,0,0,439,441,3, - 126,63,0,440,425,1,0,0,0,440,428,1,0,0,0,440,434,1,0,0,0,440,437,1,0,0, - 0,441,41,1,0,0,0,442,444,3,126,63,0,443,445,7,1,0,0,444,443,1,0,0,0,444, - 445,1,0,0,0,445,43,1,0,0,0,446,452,5,70,0,0,447,448,5,89,0,0,448,452,5, - 101,0,0,449,450,5,90,0,0,450,452,5,101,0,0,451,446,1,0,0,0,451,447,1,0, - 0,0,451,449,1,0,0,0,452,45,1,0,0,0,453,454,3,126,63,0,454,456,3,50,25, - 0,455,457,3,48,24,0,456,455,1,0,0,0,456,457,1,0,0,0,457,461,1,0,0,0,458, - 460,3,54,27,0,459,458,1,0,0,0,460,463,1,0,0,0,461,459,1,0,0,0,461,462, - 1,0,0,0,462,47,1,0,0,0,463,461,1,0,0,0,464,465,5,5,0,0,465,466,5,115,0, - 0,466,467,3,112,56,0,467,468,5,116,0,0,468,49,1,0,0,0,469,471,3,126,63, - 0,470,472,3,126,63,0,471,470,1,0,0,0,471,472,1,0,0,0,472,474,1,0,0,0,473, - 475,3,126,63,0,474,473,1,0,0,0,474,475,1,0,0,0,475,484,1,0,0,0,476,477, - 5,115,0,0,477,480,3,52,26,0,478,479,5,117,0,0,479,481,3,52,26,0,480,478, - 1,0,0,0,480,481,1,0,0,0,481,482,1,0,0,0,482,483,5,116,0,0,483,485,1,0, - 0,0,484,476,1,0,0,0,484,485,1,0,0,0,485,51,1,0,0,0,486,488,5,107,0,0,487, - 486,1,0,0,0,487,488,1,0,0,0,488,489,1,0,0,0,489,490,5,123,0,0,490,53,1, - 0,0,0,491,492,5,8,0,0,492,544,5,101,0,0,493,544,5,101,0,0,494,495,5,81, - 0,0,495,544,3,112,56,0,496,497,5,86,0,0,497,544,7,2,0,0,498,499,5,72,0, - 0,499,501,3,126,63,0,500,498,1,0,0,0,500,501,1,0,0,0,501,502,1,0,0,0,502, - 503,5,91,0,0,503,504,5,115,0,0,504,505,3,58,29,0,505,506,5,116,0,0,506, - 544,1,0,0,0,507,508,5,72,0,0,508,510,3,126,63,0,509,507,1,0,0,0,509,510, - 1,0,0,0,510,511,1,0,0,0,511,512,5,70,0,0,512,544,5,71,0,0,513,514,5,72, - 0,0,514,516,3,126,63,0,515,513,1,0,0,0,515,516,1,0,0,0,516,517,1,0,0,0, - 517,544,5,83,0,0,518,519,5,72,0,0,519,521,3,126,63,0,520,518,1,0,0,0,520, - 521,1,0,0,0,521,522,1,0,0,0,522,523,5,74,0,0,523,535,3,126,63,0,524,525, - 5,115,0,0,525,530,3,126,63,0,526,527,5,117,0,0,527,529,3,126,63,0,528, - 526,1,0,0,0,529,532,1,0,0,0,530,528,1,0,0,0,530,531,1,0,0,0,531,533,1, - 0,0,0,532,530,1,0,0,0,533,534,5,116,0,0,534,536,1,0,0,0,535,524,1,0,0, - 0,535,536,1,0,0,0,536,540,1,0,0,0,537,539,3,60,30,0,538,537,1,0,0,0,539, - 542,1,0,0,0,540,538,1,0,0,0,540,541,1,0,0,0,541,544,1,0,0,0,542,540,1, - 0,0,0,543,491,1,0,0,0,543,493,1,0,0,0,543,494,1,0,0,0,543,496,1,0,0,0, - 543,500,1,0,0,0,543,509,1,0,0,0,543,515,1,0,0,0,543,520,1,0,0,0,544,55, - 1,0,0,0,545,546,5,72,0,0,546,548,3,126,63,0,547,545,1,0,0,0,547,548,1, - 0,0,0,548,549,1,0,0,0,549,550,5,70,0,0,550,551,5,71,0,0,551,552,5,115, - 0,0,552,557,3,126,63,0,553,554,5,117,0,0,554,556,3,126,63,0,555,553,1, - 0,0,0,556,559,1,0,0,0,557,555,1,0,0,0,557,558,1,0,0,0,558,560,1,0,0,0, - 559,557,1,0,0,0,560,561,5,116,0,0,561,629,1,0,0,0,562,563,5,72,0,0,563, - 565,3,126,63,0,564,562,1,0,0,0,564,565,1,0,0,0,565,566,1,0,0,0,566,567, - 5,83,0,0,567,568,5,115,0,0,568,573,3,126,63,0,569,570,5,117,0,0,570,572, - 3,126,63,0,571,569,1,0,0,0,572,575,1,0,0,0,573,571,1,0,0,0,573,574,1,0, - 0,0,574,576,1,0,0,0,575,573,1,0,0,0,576,577,5,116,0,0,577,629,1,0,0,0, - 578,579,5,72,0,0,579,581,3,126,63,0,580,578,1,0,0,0,580,581,1,0,0,0,581, - 582,1,0,0,0,582,583,5,73,0,0,583,586,5,71,0,0,584,585,5,82,0,0,585,587, - 5,84,0,0,586,584,1,0,0,0,586,587,1,0,0,0,587,588,1,0,0,0,588,589,5,115, - 0,0,589,594,3,126,63,0,590,591,5,117,0,0,591,593,3,126,63,0,592,590,1, - 0,0,0,593,596,1,0,0,0,594,592,1,0,0,0,594,595,1,0,0,0,595,597,1,0,0,0, - 596,594,1,0,0,0,597,598,5,116,0,0,598,599,5,74,0,0,599,611,3,126,63,0, - 600,601,5,115,0,0,601,606,3,126,63,0,602,603,5,117,0,0,603,605,3,126,63, - 0,604,602,1,0,0,0,605,608,1,0,0,0,606,604,1,0,0,0,606,607,1,0,0,0,607, - 609,1,0,0,0,608,606,1,0,0,0,609,610,5,116,0,0,610,612,1,0,0,0,611,600, - 1,0,0,0,611,612,1,0,0,0,612,616,1,0,0,0,613,615,3,60,30,0,614,613,1,0, - 0,0,615,618,1,0,0,0,616,614,1,0,0,0,616,617,1,0,0,0,617,629,1,0,0,0,618, - 616,1,0,0,0,619,620,5,72,0,0,620,622,3,126,63,0,621,619,1,0,0,0,621,622, - 1,0,0,0,622,623,1,0,0,0,623,624,5,91,0,0,624,625,5,115,0,0,625,626,3,58, - 29,0,626,627,5,116,0,0,627,629,1,0,0,0,628,547,1,0,0,0,628,564,1,0,0,0, - 628,580,1,0,0,0,628,621,1,0,0,0,629,57,1,0,0,0,630,636,8,3,0,0,631,632, - 5,115,0,0,632,633,3,58,29,0,633,634,5,116,0,0,634,636,1,0,0,0,635,630, - 1,0,0,0,635,631,1,0,0,0,636,639,1,0,0,0,637,635,1,0,0,0,637,638,1,0,0, - 0,638,59,1,0,0,0,639,637,1,0,0,0,640,641,5,22,0,0,641,642,5,76,0,0,642, - 647,3,62,31,0,643,644,5,22,0,0,644,645,5,75,0,0,645,647,3,62,31,0,646, - 640,1,0,0,0,646,643,1,0,0,0,647,61,1,0,0,0,648,657,5,77,0,0,649,650,5, - 82,0,0,650,657,5,79,0,0,651,657,5,78,0,0,652,653,5,80,0,0,653,657,5,101, - 0,0,654,655,5,80,0,0,655,657,5,81,0,0,656,648,1,0,0,0,656,649,1,0,0,0, - 656,651,1,0,0,0,656,652,1,0,0,0,656,654,1,0,0,0,657,63,1,0,0,0,658,659, - 5,67,0,0,659,660,5,68,0,0,660,688,3,126,63,0,661,662,5,115,0,0,662,667, - 3,126,63,0,663,664,5,117,0,0,664,666,3,126,63,0,665,663,1,0,0,0,666,669, - 1,0,0,0,667,665,1,0,0,0,667,668,1,0,0,0,668,670,1,0,0,0,669,667,1,0,0, - 0,670,671,5,116,0,0,671,673,1,0,0,0,672,661,1,0,0,0,672,673,1,0,0,0,673, - 684,1,0,0,0,674,675,5,69,0,0,675,680,3,66,33,0,676,677,5,117,0,0,677,679, - 3,66,33,0,678,676,1,0,0,0,679,682,1,0,0,0,680,678,1,0,0,0,680,681,1,0, - 0,0,681,685,1,0,0,0,682,680,1,0,0,0,683,685,3,70,35,0,684,674,1,0,0,0, - 684,683,1,0,0,0,685,689,1,0,0,0,686,687,5,81,0,0,687,689,5,69,0,0,688, - 672,1,0,0,0,688,686,1,0,0,0,689,65,1,0,0,0,690,691,5,115,0,0,691,696,3, - 68,34,0,692,693,5,117,0,0,693,695,3,68,34,0,694,692,1,0,0,0,695,698,1, - 0,0,0,696,694,1,0,0,0,696,697,1,0,0,0,697,699,1,0,0,0,698,696,1,0,0,0, - 699,700,5,116,0,0,700,67,1,0,0,0,701,704,5,81,0,0,702,704,3,112,56,0,703, - 701,1,0,0,0,703,702,1,0,0,0,704,69,1,0,0,0,705,711,3,72,36,0,706,707,3, - 74,37,0,707,708,3,72,36,0,708,710,1,0,0,0,709,706,1,0,0,0,710,713,1,0, - 0,0,711,709,1,0,0,0,711,712,1,0,0,0,712,715,1,0,0,0,713,711,1,0,0,0,714, - 716,3,108,54,0,715,714,1,0,0,0,715,716,1,0,0,0,716,718,1,0,0,0,717,719, - 3,88,44,0,718,717,1,0,0,0,718,719,1,0,0,0,719,71,1,0,0,0,720,735,3,76, - 38,0,721,722,5,115,0,0,722,723,3,70,35,0,723,724,5,116,0,0,724,735,1,0, - 0,0,725,726,5,69,0,0,726,731,3,66,33,0,727,728,5,117,0,0,728,730,3,66, - 33,0,729,727,1,0,0,0,730,733,1,0,0,0,731,729,1,0,0,0,731,732,1,0,0,0,732, - 735,1,0,0,0,733,731,1,0,0,0,734,720,1,0,0,0,734,721,1,0,0,0,734,725,1, - 0,0,0,735,73,1,0,0,0,736,738,5,50,0,0,737,739,5,51,0,0,738,737,1,0,0,0, - 738,739,1,0,0,0,739,743,1,0,0,0,740,743,5,52,0,0,741,743,5,53,0,0,742, - 736,1,0,0,0,742,740,1,0,0,0,742,741,1,0,0,0,743,75,1,0,0,0,744,746,5,1, - 0,0,745,747,3,78,39,0,746,745,1,0,0,0,746,747,1,0,0,0,747,749,1,0,0,0, - 748,750,3,84,42,0,749,748,1,0,0,0,749,750,1,0,0,0,750,751,1,0,0,0,751, - 754,3,92,46,0,752,753,5,68,0,0,753,755,3,126,63,0,754,752,1,0,0,0,754, - 755,1,0,0,0,755,757,1,0,0,0,756,758,3,96,48,0,757,756,1,0,0,0,757,758, - 1,0,0,0,758,760,1,0,0,0,759,761,3,106,53,0,760,759,1,0,0,0,760,761,1,0, - 0,0,761,763,1,0,0,0,762,764,3,80,40,0,763,762,1,0,0,0,763,764,1,0,0,0, - 764,766,1,0,0,0,765,767,3,82,41,0,766,765,1,0,0,0,766,767,1,0,0,0,767, - 77,1,0,0,0,768,769,7,4,0,0,769,79,1,0,0,0,770,771,5,24,0,0,771,772,5,26, - 0,0,772,777,3,112,56,0,773,774,5,117,0,0,774,776,3,112,56,0,775,773,1, - 0,0,0,776,779,1,0,0,0,777,775,1,0,0,0,777,778,1,0,0,0,778,81,1,0,0,0,779, - 777,1,0,0,0,780,781,5,27,0,0,781,782,3,112,56,0,782,83,1,0,0,0,783,784, - 5,4,0,0,784,789,3,86,43,0,785,786,7,5,0,0,786,788,3,86,43,0,787,785,1, - 0,0,0,788,791,1,0,0,0,789,787,1,0,0,0,789,790,1,0,0,0,790,793,1,0,0,0, - 791,789,1,0,0,0,792,794,5,33,0,0,793,792,1,0,0,0,793,794,1,0,0,0,794,85, - 1,0,0,0,795,802,5,123,0,0,796,802,5,121,0,0,797,798,5,115,0,0,798,799, - 3,112,56,0,799,800,5,116,0,0,800,802,1,0,0,0,801,795,1,0,0,0,801,796,1, - 0,0,0,801,797,1,0,0,0,802,87,1,0,0,0,803,804,5,42,0,0,804,805,3,112,56, - 0,805,812,3,90,45,0,806,807,5,43,0,0,807,808,7,6,0,0,808,809,3,112,56, - 0,809,810,3,90,45,0,810,811,5,48,0,0,811,813,1,0,0,0,812,806,1,0,0,0,812, - 813,1,0,0,0,813,821,1,0,0,0,814,815,5,43,0,0,815,816,7,6,0,0,816,817,3, - 112,56,0,817,818,3,90,45,0,818,819,5,48,0,0,819,821,1,0,0,0,820,803,1, - 0,0,0,820,814,1,0,0,0,821,89,1,0,0,0,822,823,7,7,0,0,823,91,1,0,0,0,824, - 834,5,102,0,0,825,830,3,94,47,0,826,827,5,117,0,0,827,829,3,94,47,0,828, - 826,1,0,0,0,829,832,1,0,0,0,830,828,1,0,0,0,830,831,1,0,0,0,831,834,1, - 0,0,0,832,830,1,0,0,0,833,824,1,0,0,0,833,825,1,0,0,0,834,93,1,0,0,0,835, - 836,3,126,63,0,836,837,5,118,0,0,837,838,5,102,0,0,838,847,1,0,0,0,839, - 844,3,112,56,0,840,842,5,5,0,0,841,840,1,0,0,0,841,842,1,0,0,0,842,843, - 1,0,0,0,843,845,3,126,63,0,844,841,1,0,0,0,844,845,1,0,0,0,845,847,1,0, - 0,0,846,835,1,0,0,0,846,839,1,0,0,0,847,95,1,0,0,0,848,849,5,2,0,0,849, - 854,3,98,49,0,850,851,5,117,0,0,851,853,3,98,49,0,852,850,1,0,0,0,853, - 856,1,0,0,0,854,852,1,0,0,0,854,855,1,0,0,0,855,97,1,0,0,0,856,854,1,0, - 0,0,857,861,3,100,50,0,858,860,3,102,51,0,859,858,1,0,0,0,860,863,1,0, - 0,0,861,859,1,0,0,0,861,862,1,0,0,0,862,99,1,0,0,0,863,861,1,0,0,0,864, - 869,3,126,63,0,865,867,5,5,0,0,866,865,1,0,0,0,866,867,1,0,0,0,867,868, - 1,0,0,0,868,870,3,126,63,0,869,866,1,0,0,0,869,870,1,0,0,0,870,885,1,0, - 0,0,871,872,5,115,0,0,872,873,3,70,35,0,873,878,5,116,0,0,874,876,5,5, - 0,0,875,874,1,0,0,0,875,876,1,0,0,0,876,877,1,0,0,0,877,879,3,126,63,0, - 878,875,1,0,0,0,878,879,1,0,0,0,879,885,1,0,0,0,880,881,5,115,0,0,881, - 882,3,98,49,0,882,883,5,116,0,0,883,885,1,0,0,0,884,864,1,0,0,0,884,871, - 1,0,0,0,884,880,1,0,0,0,885,101,1,0,0,0,886,887,3,104,52,0,887,888,5,20, - 0,0,888,889,3,100,50,0,889,890,5,22,0,0,890,891,3,112,56,0,891,902,1,0, - 0,0,892,893,5,34,0,0,893,894,5,20,0,0,894,902,3,100,50,0,895,896,5,34, - 0,0,896,897,5,35,0,0,897,902,3,100,50,0,898,899,5,19,0,0,899,900,5,35, - 0,0,900,902,3,100,50,0,901,886,1,0,0,0,901,892,1,0,0,0,901,895,1,0,0,0, - 901,898,1,0,0,0,902,103,1,0,0,0,903,905,5,15,0,0,904,903,1,0,0,0,904,905, - 1,0,0,0,905,919,1,0,0,0,906,908,5,16,0,0,907,909,5,19,0,0,908,907,1,0, - 0,0,908,909,1,0,0,0,909,919,1,0,0,0,910,912,5,17,0,0,911,913,5,19,0,0, - 912,911,1,0,0,0,912,913,1,0,0,0,913,919,1,0,0,0,914,916,5,18,0,0,915,917, - 5,19,0,0,916,915,1,0,0,0,916,917,1,0,0,0,917,919,1,0,0,0,918,904,1,0,0, - 0,918,906,1,0,0,0,918,910,1,0,0,0,918,914,1,0,0,0,919,105,1,0,0,0,920, - 921,5,3,0,0,921,922,3,112,56,0,922,107,1,0,0,0,923,924,5,23,0,0,924,925, - 5,26,0,0,925,930,3,110,55,0,926,927,5,117,0,0,927,929,3,110,55,0,928,926, - 1,0,0,0,929,932,1,0,0,0,930,928,1,0,0,0,930,931,1,0,0,0,931,109,1,0,0, - 0,932,930,1,0,0,0,933,935,3,112,56,0,934,936,7,1,0,0,935,934,1,0,0,0,935, - 936,1,0,0,0,936,111,1,0,0,0,937,938,6,56,-1,0,938,939,5,8,0,0,939,946, - 3,112,56,16,940,941,5,12,0,0,941,946,3,112,56,15,942,943,5,107,0,0,943, - 946,3,112,56,14,944,946,3,114,57,0,945,937,1,0,0,0,945,940,1,0,0,0,945, - 942,1,0,0,0,945,944,1,0,0,0,946,1016,1,0,0,0,947,948,10,13,0,0,948,949, - 5,105,0,0,949,1015,3,112,56,14,950,951,10,12,0,0,951,952,7,8,0,0,952,1015, - 3,112,56,13,953,954,10,11,0,0,954,955,7,9,0,0,955,1015,3,112,56,12,956, - 957,10,10,0,0,957,958,7,10,0,0,958,1015,3,112,56,11,959,961,10,9,0,0,960, - 962,5,8,0,0,961,960,1,0,0,0,961,962,1,0,0,0,962,963,1,0,0,0,963,964,5, - 49,0,0,964,965,3,112,56,0,965,966,5,6,0,0,966,967,3,112,56,10,967,1015, - 1,0,0,0,968,970,10,8,0,0,969,971,5,8,0,0,970,969,1,0,0,0,970,971,1,0,0, - 0,971,972,1,0,0,0,972,973,5,13,0,0,973,1015,3,112,56,9,974,975,10,4,0, - 0,975,976,7,11,0,0,976,1015,3,112,56,5,977,978,10,3,0,0,978,979,5,6,0, - 0,979,1015,3,112,56,4,980,981,10,2,0,0,981,982,5,7,0,0,982,1015,3,112, - 56,3,983,985,10,7,0,0,984,986,5,8,0,0,985,984,1,0,0,0,985,986,1,0,0,0, - 986,987,1,0,0,0,987,988,5,21,0,0,988,989,5,115,0,0,989,990,3,70,35,0,990, - 991,5,116,0,0,991,1015,1,0,0,0,992,994,10,6,0,0,993,995,5,8,0,0,994,993, - 1,0,0,0,994,995,1,0,0,0,995,996,1,0,0,0,996,997,5,21,0,0,997,998,5,115, - 0,0,998,1003,3,112,56,0,999,1000,5,117,0,0,1000,1002,3,112,56,0,1001,999, - 1,0,0,0,1002,1005,1,0,0,0,1003,1001,1,0,0,0,1003,1004,1,0,0,0,1004,1006, - 1,0,0,0,1005,1003,1,0,0,0,1006,1007,5,116,0,0,1007,1015,1,0,0,0,1008,1009, - 10,5,0,0,1009,1011,5,25,0,0,1010,1012,5,8,0,0,1011,1010,1,0,0,0,1011,1012, - 1,0,0,0,1012,1013,1,0,0,0,1013,1015,5,101,0,0,1014,947,1,0,0,0,1014,950, - 1,0,0,0,1014,953,1,0,0,0,1014,956,1,0,0,0,1014,959,1,0,0,0,1014,968,1, - 0,0,0,1014,974,1,0,0,0,1014,977,1,0,0,0,1014,980,1,0,0,0,1014,983,1,0, - 0,0,1014,992,1,0,0,0,1014,1008,1,0,0,0,1015,1018,1,0,0,0,1016,1014,1,0, - 0,0,1016,1017,1,0,0,0,1017,113,1,0,0,0,1018,1016,1,0,0,0,1019,1039,3,128, - 64,0,1020,1039,3,116,58,0,1021,1039,3,120,60,0,1022,1039,3,124,62,0,1023, - 1039,5,121,0,0,1024,1039,5,120,0,0,1025,1026,5,28,0,0,1026,1027,5,115, - 0,0,1027,1028,3,70,35,0,1028,1029,5,116,0,0,1029,1039,1,0,0,0,1030,1031, - 5,115,0,0,1031,1032,3,70,35,0,1032,1033,5,116,0,0,1033,1039,1,0,0,0,1034, - 1035,5,115,0,0,1035,1036,3,112,56,0,1036,1037,5,116,0,0,1037,1039,1,0, - 0,0,1038,1019,1,0,0,0,1038,1020,1,0,0,0,1038,1021,1,0,0,0,1038,1022,1, - 0,0,0,1038,1023,1,0,0,0,1038,1024,1,0,0,0,1038,1025,1,0,0,0,1038,1030, - 1,0,0,0,1038,1034,1,0,0,0,1039,115,1,0,0,0,1040,1042,5,38,0,0,1041,1043, - 3,112,56,0,1042,1041,1,0,0,0,1042,1043,1,0,0,0,1043,1045,1,0,0,0,1044, - 1046,3,118,59,0,1045,1044,1,0,0,0,1046,1047,1,0,0,0,1047,1045,1,0,0,0, - 1047,1048,1,0,0,0,1048,1051,1,0,0,0,1049,1050,5,40,0,0,1050,1052,3,112, - 56,0,1051,1049,1,0,0,0,1051,1052,1,0,0,0,1052,1053,1,0,0,0,1053,1054,5, - 41,0,0,1054,117,1,0,0,0,1055,1056,5,39,0,0,1056,1057,3,112,56,0,1057,1058, - 5,30,0,0,1058,1059,3,112,56,0,1059,119,1,0,0,0,1060,1061,3,122,61,0,1061, - 1074,5,115,0,0,1062,1075,5,102,0,0,1063,1065,5,32,0,0,1064,1063,1,0,0, - 0,1064,1065,1,0,0,0,1065,1066,1,0,0,0,1066,1071,3,112,56,0,1067,1068,5, - 117,0,0,1068,1070,3,112,56,0,1069,1067,1,0,0,0,1070,1073,1,0,0,0,1071, - 1069,1,0,0,0,1071,1072,1,0,0,0,1072,1075,1,0,0,0,1073,1071,1,0,0,0,1074, - 1062,1,0,0,0,1074,1064,1,0,0,0,1074,1075,1,0,0,0,1075,1076,1,0,0,0,1076, - 1079,5,116,0,0,1077,1078,5,36,0,0,1078,1080,3,134,67,0,1079,1077,1,0,0, - 0,1079,1080,1,0,0,0,1080,121,1,0,0,0,1081,1088,3,126,63,0,1082,1088,5, - 16,0,0,1083,1088,5,17,0,0,1084,1088,5,97,0,0,1085,1088,5,45,0,0,1086,1088, - 5,37,0,0,1087,1081,1,0,0,0,1087,1082,1,0,0,0,1087,1083,1,0,0,0,1087,1084, - 1,0,0,0,1087,1085,1,0,0,0,1087,1086,1,0,0,0,1088,123,1,0,0,0,1089,1090, - 3,126,63,0,1090,1091,5,118,0,0,1091,1093,1,0,0,0,1092,1089,1,0,0,0,1092, - 1093,1,0,0,0,1093,1094,1,0,0,0,1094,1095,3,126,63,0,1095,125,1,0,0,0,1096, - 1097,7,12,0,0,1097,127,1,0,0,0,1098,1108,5,123,0,0,1099,1108,5,124,0,0, - 1100,1108,5,122,0,0,1101,1108,5,125,0,0,1102,1108,5,126,0,0,1103,1108, - 5,127,0,0,1104,1108,5,99,0,0,1105,1108,5,100,0,0,1106,1108,5,101,0,0,1107, - 1098,1,0,0,0,1107,1099,1,0,0,0,1107,1100,1,0,0,0,1107,1101,1,0,0,0,1107, - 1102,1,0,0,0,1107,1103,1,0,0,0,1107,1104,1,0,0,0,1107,1105,1,0,0,0,1107, - 1106,1,0,0,0,1108,129,1,0,0,0,1109,1111,5,56,0,0,1110,1112,7,13,0,0,1111, - 1110,1,0,0,0,1111,1112,1,0,0,0,1112,1122,1,0,0,0,1113,1115,5,57,0,0,1114, - 1116,7,13,0,0,1115,1114,1,0,0,0,1115,1116,1,0,0,0,1116,1122,1,0,0,0,1117, - 1119,5,58,0,0,1118,1120,7,13,0,0,1119,1118,1,0,0,0,1119,1120,1,0,0,0,1120, - 1122,1,0,0,0,1121,1109,1,0,0,0,1121,1113,1,0,0,0,1121,1117,1,0,0,0,1122, - 131,1,0,0,0,1123,1124,3,112,56,0,1124,1125,5,0,0,1,1125,133,1,0,0,0,1126, - 1137,5,115,0,0,1127,1128,5,37,0,0,1128,1129,5,26,0,0,1129,1134,3,112,56, - 0,1130,1131,5,117,0,0,1131,1133,3,112,56,0,1132,1130,1,0,0,0,1133,1136, - 1,0,0,0,1134,1132,1,0,0,0,1134,1135,1,0,0,0,1135,1138,1,0,0,0,1136,1134, - 1,0,0,0,1137,1127,1,0,0,0,1137,1138,1,0,0,0,1138,1140,1,0,0,0,1139,1141, - 3,108,54,0,1140,1139,1,0,0,0,1140,1141,1,0,0,0,1141,1142,1,0,0,0,1142, - 1143,5,116,0,0,1143,135,1,0,0,0,143,137,153,156,162,182,191,194,204,208, - 220,225,233,238,241,249,256,266,273,287,292,301,312,322,325,332,341,348, - 355,360,369,396,401,405,417,423,440,444,451,456,461,471,474,480,484,487, - 500,509,515,520,530,535,540,543,547,557,564,573,580,586,594,606,611,616, - 621,628,635,637,646,656,667,672,680,684,688,696,703,711,715,718,731,734, - 738,742,746,749,754,757,760,763,766,777,789,793,801,812,820,830,833,841, - 844,846,854,861,866,869,875,878,884,901,904,908,912,916,918,930,935,945, - 961,970,985,994,1003,1011,1014,1016,1038,1042,1047,1051,1064,1071,1074, - 1079,1087,1092,1107,1111,1115,1119,1121,1134,1137,1140 + 2,64,7,64,2,65,7,65,2,66,7,66,2,67,7,67,2,68,7,68,2,69,7,69,2,70,7,70, + 2,71,7,71,2,72,7,72,2,73,7,73,2,74,7,74,2,75,7,75,2,76,7,76,1,0,3,0,156, + 8,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,3,0,172,8, + 0,1,0,3,0,175,8,0,1,0,1,0,1,1,1,1,3,1,181,8,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,3,2,201,8,2,1,3,1,3,1, + 3,1,3,1,3,5,3,208,8,3,10,3,12,3,211,9,3,3,3,213,8,3,1,4,1,4,1,4,1,4,5, + 4,219,8,4,10,4,12,4,222,9,4,1,4,1,4,1,4,1,4,5,4,228,8,4,10,4,12,4,231, + 9,4,1,4,3,4,234,8,4,1,5,1,5,1,5,1,5,1,6,1,6,1,6,1,6,1,6,1,6,3,6,246,8, + 6,1,6,1,6,1,6,1,6,5,6,252,8,6,10,6,12,6,255,9,6,1,6,3,6,258,8,6,1,7,1, + 7,1,7,1,7,5,7,264,8,7,10,7,12,7,267,9,7,1,8,1,8,3,8,271,8,8,1,8,3,8,274, + 8,8,1,9,1,9,1,9,1,9,5,9,280,8,9,10,9,12,9,283,9,9,1,9,1,9,1,10,1,10,3, + 10,289,8,10,1,10,1,10,1,10,1,10,1,10,1,10,5,10,297,8,10,10,10,12,10,300, + 9,10,1,10,1,10,5,10,304,8,10,10,10,12,10,307,9,10,1,10,1,10,1,11,1,11, + 1,11,1,11,1,11,1,11,1,11,5,11,318,8,11,10,11,12,11,321,9,11,1,11,1,11, + 3,11,325,8,11,1,11,1,11,1,11,1,12,1,12,1,12,1,12,3,12,334,8,12,1,12,1, + 12,1,12,1,13,1,13,1,13,1,13,5,13,343,8,13,10,13,12,13,346,9,13,1,13,1, + 13,1,13,1,13,1,13,5,13,353,8,13,10,13,12,13,356,9,13,3,13,358,8,13,1,14, + 1,14,1,14,1,15,1,15,3,15,365,8,15,1,16,1,16,1,16,1,16,1,16,1,17,1,17,3, + 17,374,8,17,1,17,1,17,1,17,1,17,1,17,3,17,381,8,17,1,17,1,17,1,17,5,17, + 386,8,17,10,17,12,17,389,9,17,1,17,1,17,3,17,393,8,17,1,17,1,17,1,17,1, + 17,1,17,1,17,1,17,3,17,402,8,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17, + 1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17,1,17, + 1,17,1,17,1,17,3,17,429,8,17,1,18,1,18,1,18,3,18,434,8,18,1,19,1,19,3, + 19,438,8,19,1,19,1,19,1,19,1,19,1,19,1,19,1,19,1,19,5,19,448,8,19,10,19, + 12,19,451,9,19,1,19,1,19,1,19,3,19,456,8,19,1,20,1,20,1,20,1,20,1,20,1, + 20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,1,20,3,20,473,8,20,1,21,1,21, + 3,21,477,8,21,1,22,1,22,1,22,1,22,1,22,3,22,484,8,22,1,23,1,23,1,23,3, + 23,489,8,23,1,23,5,23,492,8,23,10,23,12,23,495,9,23,1,24,1,24,1,24,1,24, + 1,24,1,25,1,25,3,25,504,8,25,1,25,3,25,507,8,25,1,25,3,25,510,8,25,1,25, + 1,25,1,25,1,25,3,25,516,8,25,1,25,1,25,3,25,520,8,25,1,26,3,26,523,8,26, + 1,26,1,26,1,27,1,27,1,27,1,27,1,27,1,27,1,27,1,27,1,27,3,27,536,8,27,1, + 27,1,27,1,27,1,27,1,27,1,27,1,27,3,27,545,8,27,1,27,1,27,1,27,3,27,550, + 8,27,1,27,1,27,3,27,554,8,27,1,27,1,27,3,27,558,8,27,1,27,1,27,3,27,562, + 8,27,1,27,1,27,1,27,1,27,1,27,1,27,5,27,570,8,27,10,27,12,27,573,9,27, + 1,27,1,27,3,27,577,8,27,1,27,5,27,580,8,27,10,27,12,27,583,9,27,1,27,1, + 27,1,27,1,27,1,27,3,27,590,8,27,1,27,1,27,3,27,594,8,27,3,27,596,8,27, + 1,28,1,28,3,28,600,8,28,1,28,1,28,1,28,3,28,605,8,28,1,28,1,28,1,28,1, + 28,5,28,611,8,28,10,28,12,28,614,9,28,1,28,1,28,1,28,1,28,3,28,620,8,28, + 1,28,1,28,3,28,624,8,28,1,28,1,28,1,28,1,28,5,28,630,8,28,10,28,12,28, + 633,9,28,1,28,1,28,1,28,1,28,3,28,639,8,28,1,28,1,28,1,28,1,28,3,28,645, + 8,28,1,28,1,28,1,28,1,28,5,28,651,8,28,10,28,12,28,654,9,28,1,28,1,28, + 1,28,1,28,1,28,1,28,1,28,5,28,663,8,28,10,28,12,28,666,9,28,1,28,1,28, + 3,28,670,8,28,1,28,5,28,673,8,28,10,28,12,28,676,9,28,1,28,1,28,3,28,680, + 8,28,1,28,1,28,1,28,1,28,1,28,3,28,687,8,28,1,29,1,29,1,29,1,29,1,29,5, + 29,694,8,29,10,29,12,29,697,9,29,1,30,1,30,1,30,1,30,1,30,1,30,3,30,705, + 8,30,1,31,1,31,1,31,1,31,1,31,1,31,1,31,1,31,3,31,715,8,31,1,32,1,32,1, + 32,1,32,1,32,1,32,1,32,5,32,724,8,32,10,32,12,32,727,9,32,1,32,1,32,3, + 32,731,8,32,1,32,1,32,1,32,1,32,5,32,737,8,32,10,32,12,32,740,9,32,1,32, + 3,32,743,8,32,1,32,1,32,3,32,747,8,32,1,33,1,33,1,33,1,33,5,33,753,8,33, + 10,33,12,33,756,9,33,1,33,1,33,1,34,1,34,3,34,762,8,34,1,35,1,35,1,35, + 1,35,5,35,768,8,35,10,35,12,35,771,9,35,1,35,3,35,774,8,35,1,35,3,35,777, + 8,35,1,36,1,36,1,36,1,36,1,36,1,36,1,36,1,36,1,36,5,36,788,8,36,10,36, + 12,36,791,9,36,3,36,793,8,36,1,37,1,37,3,37,797,8,37,1,37,1,37,3,37,801, + 8,37,1,38,1,38,3,38,805,8,38,1,38,3,38,808,8,38,1,38,1,38,1,38,3,38,813, + 8,38,1,38,3,38,816,8,38,1,38,3,38,819,8,38,1,38,3,38,822,8,38,1,38,3,38, + 825,8,38,1,39,1,39,1,40,1,40,1,40,1,40,1,40,5,40,834,8,40,10,40,12,40, + 837,9,40,1,41,1,41,1,41,1,42,1,42,1,42,1,42,5,42,846,8,42,10,42,12,42, + 849,9,42,1,42,3,42,852,8,42,1,43,1,43,1,43,1,43,1,43,1,43,3,43,860,8,43, + 1,44,1,44,1,44,1,44,1,44,1,44,1,44,1,44,1,44,3,44,871,8,44,1,44,1,44,1, + 44,1,44,1,44,1,44,3,44,879,8,44,1,45,1,45,1,46,1,46,1,46,1,46,5,46,887, + 8,46,10,46,12,46,890,9,46,3,46,892,8,46,1,47,1,47,1,47,1,47,1,47,1,47, + 3,47,900,8,47,1,47,3,47,903,8,47,3,47,905,8,47,1,48,1,48,1,48,1,48,5,48, + 911,8,48,10,48,12,48,914,9,48,1,49,1,49,5,49,918,8,49,10,49,12,49,921, + 9,49,1,50,1,50,3,50,925,8,50,1,50,3,50,928,8,50,1,50,1,50,1,50,1,50,3, + 50,934,8,50,1,50,3,50,937,8,50,1,50,1,50,1,50,1,50,3,50,943,8,50,1,51, + 1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51,1,51, + 3,51,960,8,51,1,52,3,52,963,8,52,1,52,1,52,3,52,967,8,52,1,52,1,52,3,52, + 971,8,52,1,52,1,52,3,52,975,8,52,3,52,977,8,52,1,53,1,53,1,53,1,54,1,54, + 1,54,1,54,1,54,5,54,987,8,54,10,54,12,54,990,9,54,1,55,1,55,3,55,994,8, + 55,1,56,1,56,1,56,1,56,1,56,1,56,3,56,1002,8,56,1,56,1,56,1,56,1,56,1, + 56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1, + 56,1,56,1,56,1,56,1,56,3,56,1027,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1, + 56,3,56,1036,8,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1, + 56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,1,56,3,56,1057,8,56,1,56,1,56,1, + 56,1,56,1,56,1,56,1,56,3,56,1066,8,56,1,56,1,56,1,56,1,56,1,56,5,56,1073, + 8,56,10,56,12,56,1076,9,56,1,56,1,56,1,56,1,56,1,56,3,56,1083,8,56,1,56, + 5,56,1086,8,56,10,56,12,56,1089,9,56,1,57,1,57,1,57,1,57,1,57,1,57,1,57, + 1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,1,57,3,57,1110, + 8,57,1,58,1,58,3,58,1114,8,58,1,58,4,58,1117,8,58,11,58,12,58,1118,1,58, + 1,58,3,58,1123,8,58,1,58,1,58,1,59,1,59,1,59,1,59,1,59,1,60,1,60,1,60, + 1,60,3,60,1136,8,60,1,60,1,60,1,60,5,60,1141,8,60,10,60,12,60,1144,9,60, + 3,60,1146,8,60,1,60,1,60,3,60,1150,8,60,1,60,3,60,1153,8,60,1,60,3,60, + 1156,8,60,1,60,3,60,1159,8,60,1,60,1,60,3,60,1163,8,60,1,61,1,61,1,61, + 1,61,1,61,1,61,3,61,1171,8,61,1,62,1,62,1,62,3,62,1176,8,62,1,62,1,62, + 1,63,1,63,1,63,1,63,3,63,1184,8,63,1,64,1,64,1,64,1,64,1,64,1,64,1,64, + 1,64,1,64,3,64,1195,8,64,1,65,1,65,3,65,1199,8,65,1,65,1,65,3,65,1203, + 8,65,1,65,1,65,3,65,1207,8,65,3,65,1209,8,65,1,66,1,66,1,66,1,67,1,67, + 1,67,1,67,1,67,1,67,5,67,1220,8,67,10,67,12,67,1223,9,67,3,67,1225,8,67, + 1,67,3,67,1228,8,67,1,67,3,67,1231,8,67,1,67,1,67,1,68,1,68,1,69,1,69, + 1,69,1,69,1,69,1,69,1,69,3,69,1244,8,69,1,69,1,69,3,69,1248,8,69,1,70, + 1,70,1,70,1,70,1,70,1,70,1,70,3,70,1257,8,70,1,71,1,71,1,71,1,71,1,71, + 1,71,3,71,1265,8,71,1,72,1,72,1,73,1,73,1,73,1,73,1,73,1,73,1,74,1,74, + 1,74,1,74,1,74,1,74,1,75,1,75,1,75,1,76,1,76,1,76,1,76,0,1,112,77,0,2, + 4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52, + 54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100, + 102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136, + 138,140,142,144,146,148,150,152,0,21,1,0,101,102,1,0,103,104,1,0,93,94, + 1,0,135,136,2,0,34,35,54,54,1,0,126,127,1,0,47,48,1,0,49,50,2,0,8,8,15, + 15,1,0,122,123,1,0,129,134,2,0,6,6,12,12,2,0,7,7,13,13,2,0,9,9,14,14,1, + 0,62,63,2,0,87,87,89,89,2,0,49,49,108,109,1,0,111,112,1,0,108,121,2,0, + 48,48,118,118,2,0,96,96,119,119,1466,0,155,1,0,0,0,2,178,1,0,0,0,4,200, + 1,0,0,0,6,202,1,0,0,0,8,214,1,0,0,0,10,235,1,0,0,0,12,239,1,0,0,0,14,259, + 1,0,0,0,16,268,1,0,0,0,18,275,1,0,0,0,20,286,1,0,0,0,22,310,1,0,0,0,24, + 329,1,0,0,0,26,357,1,0,0,0,28,359,1,0,0,0,30,364,1,0,0,0,32,366,1,0,0, + 0,34,428,1,0,0,0,36,433,1,0,0,0,38,435,1,0,0,0,40,472,1,0,0,0,42,474,1, + 0,0,0,44,483,1,0,0,0,46,485,1,0,0,0,48,496,1,0,0,0,50,503,1,0,0,0,52,522, + 1,0,0,0,54,595,1,0,0,0,56,686,1,0,0,0,58,695,1,0,0,0,60,704,1,0,0,0,62, + 714,1,0,0,0,64,716,1,0,0,0,66,748,1,0,0,0,68,761,1,0,0,0,70,763,1,0,0, + 0,72,792,1,0,0,0,74,800,1,0,0,0,76,802,1,0,0,0,78,826,1,0,0,0,80,828,1, + 0,0,0,82,838,1,0,0,0,84,841,1,0,0,0,86,859,1,0,0,0,88,878,1,0,0,0,90,880, + 1,0,0,0,92,891,1,0,0,0,94,904,1,0,0,0,96,906,1,0,0,0,98,915,1,0,0,0,100, + 942,1,0,0,0,102,959,1,0,0,0,104,976,1,0,0,0,106,978,1,0,0,0,108,981,1, + 0,0,0,110,991,1,0,0,0,112,1001,1,0,0,0,114,1109,1,0,0,0,116,1111,1,0,0, + 0,118,1126,1,0,0,0,120,1131,1,0,0,0,122,1170,1,0,0,0,124,1175,1,0,0,0, + 126,1183,1,0,0,0,128,1194,1,0,0,0,130,1208,1,0,0,0,132,1210,1,0,0,0,134, + 1213,1,0,0,0,136,1234,1,0,0,0,138,1236,1,0,0,0,140,1256,1,0,0,0,142,1264, + 1,0,0,0,144,1266,1,0,0,0,146,1268,1,0,0,0,148,1274,1,0,0,0,150,1280,1, + 0,0,0,152,1283,1,0,0,0,154,156,3,18,9,0,155,154,1,0,0,0,155,156,1,0,0, + 0,156,171,1,0,0,0,157,172,3,2,1,0,158,172,3,20,10,0,159,172,3,38,19,0, + 160,172,3,22,11,0,161,172,3,24,12,0,162,172,3,32,16,0,163,172,3,40,20, + 0,164,172,3,64,32,0,165,172,3,8,4,0,166,172,3,12,6,0,167,172,3,130,65, + 0,168,172,3,6,3,0,169,172,3,14,7,0,170,172,3,70,35,0,171,157,1,0,0,0,171, + 158,1,0,0,0,171,159,1,0,0,0,171,160,1,0,0,0,171,161,1,0,0,0,171,162,1, + 0,0,0,171,163,1,0,0,0,171,164,1,0,0,0,171,165,1,0,0,0,171,166,1,0,0,0, + 171,167,1,0,0,0,171,168,1,0,0,0,171,169,1,0,0,0,171,170,1,0,0,0,172,174, + 1,0,0,0,173,175,5,139,0,0,174,173,1,0,0,0,174,175,1,0,0,0,175,176,1,0, + 0,0,176,177,5,0,0,1,177,1,1,0,0,0,178,180,5,32,0,0,179,181,5,8,0,0,180, + 179,1,0,0,0,180,181,1,0,0,0,181,182,1,0,0,0,182,183,5,31,0,0,183,184,5, + 135,0,0,184,185,3,70,35,0,185,186,5,136,0,0,186,187,5,33,0,0,187,188,3, + 4,2,0,188,3,1,0,0,0,189,201,3,20,10,0,190,201,3,38,19,0,191,201,3,22,11, + 0,192,201,3,24,12,0,193,201,3,32,16,0,194,201,3,40,20,0,195,201,3,64,32, + 0,196,201,3,8,4,0,197,201,3,12,6,0,198,201,3,6,3,0,199,201,3,70,35,0,200, + 189,1,0,0,0,200,190,1,0,0,0,200,191,1,0,0,0,200,192,1,0,0,0,200,193,1, + 0,0,0,200,194,1,0,0,0,200,195,1,0,0,0,200,196,1,0,0,0,200,197,1,0,0,0, + 200,198,1,0,0,0,200,199,1,0,0,0,201,5,1,0,0,0,202,203,7,0,0,0,203,212, + 3,126,63,0,204,209,3,112,56,0,205,206,5,137,0,0,206,208,3,112,56,0,207, + 205,1,0,0,0,208,211,1,0,0,0,209,207,1,0,0,0,209,210,1,0,0,0,210,213,1, + 0,0,0,211,209,1,0,0,0,212,204,1,0,0,0,212,213,1,0,0,0,213,7,1,0,0,0,214, + 215,5,79,0,0,215,220,3,98,49,0,216,217,5,137,0,0,217,219,3,98,49,0,218, + 216,1,0,0,0,219,222,1,0,0,0,220,218,1,0,0,0,220,221,1,0,0,0,221,223,1, + 0,0,0,222,220,1,0,0,0,223,224,5,83,0,0,224,229,3,10,5,0,225,226,5,137, + 0,0,226,228,3,10,5,0,227,225,1,0,0,0,228,231,1,0,0,0,229,227,1,0,0,0,229, + 230,1,0,0,0,230,233,1,0,0,0,231,229,1,0,0,0,232,234,3,106,53,0,233,232, + 1,0,0,0,233,234,1,0,0,0,234,9,1,0,0,0,235,236,3,124,62,0,236,237,5,129, + 0,0,237,238,3,112,56,0,238,11,1,0,0,0,239,245,5,78,0,0,240,241,3,126,63, + 0,241,242,5,138,0,0,242,243,5,122,0,0,243,246,1,0,0,0,244,246,5,122,0, + 0,245,240,1,0,0,0,245,244,1,0,0,0,245,246,1,0,0,0,246,247,1,0,0,0,247, + 248,5,2,0,0,248,253,3,98,49,0,249,250,5,137,0,0,250,252,3,98,49,0,251, + 249,1,0,0,0,252,255,1,0,0,0,253,251,1,0,0,0,253,254,1,0,0,0,254,257,1, + 0,0,0,255,253,1,0,0,0,256,258,3,106,53,0,257,256,1,0,0,0,257,258,1,0,0, + 0,258,13,1,0,0,0,259,260,5,1,0,0,260,265,3,16,8,0,261,262,5,137,0,0,262, + 264,3,16,8,0,263,261,1,0,0,0,264,267,1,0,0,0,265,263,1,0,0,0,265,266,1, + 0,0,0,266,15,1,0,0,0,267,265,1,0,0,0,268,273,5,140,0,0,269,271,5,5,0,0, + 270,269,1,0,0,0,270,271,1,0,0,0,271,272,1,0,0,0,272,274,3,126,63,0,273, + 270,1,0,0,0,273,274,1,0,0,0,274,17,1,0,0,0,275,276,5,100,0,0,276,281,3, + 28,14,0,277,278,5,137,0,0,278,280,3,28,14,0,279,277,1,0,0,0,280,283,1, + 0,0,0,281,279,1,0,0,0,281,282,1,0,0,0,282,284,1,0,0,0,283,281,1,0,0,0, + 284,285,5,139,0,0,285,19,1,0,0,0,286,288,5,57,0,0,287,289,5,91,0,0,288, + 287,1,0,0,0,288,289,1,0,0,0,289,290,1,0,0,0,290,291,5,58,0,0,291,292,3, + 126,63,0,292,293,5,135,0,0,293,298,3,46,23,0,294,295,5,137,0,0,295,297, + 3,46,23,0,296,294,1,0,0,0,297,300,1,0,0,0,298,296,1,0,0,0,298,299,1,0, + 0,0,299,305,1,0,0,0,300,298,1,0,0,0,301,302,5,137,0,0,302,304,3,56,28, + 0,303,301,1,0,0,0,304,307,1,0,0,0,305,303,1,0,0,0,305,306,1,0,0,0,306, + 308,1,0,0,0,307,305,1,0,0,0,308,309,5,136,0,0,309,21,1,0,0,0,310,311,5, + 57,0,0,311,312,5,98,0,0,312,324,3,126,63,0,313,314,5,135,0,0,314,319,3, + 126,63,0,315,316,5,137,0,0,316,318,3,126,63,0,317,315,1,0,0,0,318,321, + 1,0,0,0,319,317,1,0,0,0,319,320,1,0,0,0,320,322,1,0,0,0,321,319,1,0,0, + 0,322,323,5,136,0,0,323,325,1,0,0,0,324,313,1,0,0,0,324,325,1,0,0,0,325, + 326,1,0,0,0,326,327,5,5,0,0,327,328,3,70,35,0,328,23,1,0,0,0,329,330,5, + 57,0,0,330,331,5,99,0,0,331,333,3,126,63,0,332,334,3,26,13,0,333,332,1, + 0,0,0,333,334,1,0,0,0,334,335,1,0,0,0,335,336,5,5,0,0,336,337,3,36,18, + 0,337,25,1,0,0,0,338,339,5,135,0,0,339,344,3,28,14,0,340,341,5,137,0,0, + 341,343,3,28,14,0,342,340,1,0,0,0,343,346,1,0,0,0,344,342,1,0,0,0,344, + 345,1,0,0,0,345,347,1,0,0,0,346,344,1,0,0,0,347,348,5,136,0,0,348,358, + 1,0,0,0,349,354,3,28,14,0,350,351,5,137,0,0,351,353,3,28,14,0,352,350, + 1,0,0,0,353,356,1,0,0,0,354,352,1,0,0,0,354,355,1,0,0,0,355,358,1,0,0, + 0,356,354,1,0,0,0,357,338,1,0,0,0,357,349,1,0,0,0,358,27,1,0,0,0,359,360, + 3,30,15,0,360,361,3,50,25,0,361,29,1,0,0,0,362,365,3,126,63,0,363,365, + 5,141,0,0,364,362,1,0,0,0,364,363,1,0,0,0,365,31,1,0,0,0,366,367,5,64, + 0,0,367,368,5,58,0,0,368,369,3,126,63,0,369,370,3,34,17,0,370,33,1,0,0, + 0,371,373,5,67,0,0,372,374,5,69,0,0,373,372,1,0,0,0,373,374,1,0,0,0,374, + 375,1,0,0,0,375,429,3,46,23,0,376,377,5,67,0,0,377,429,3,56,28,0,378,380, + 5,64,0,0,379,381,5,69,0,0,380,379,1,0,0,0,380,381,1,0,0,0,381,382,1,0, + 0,0,382,383,3,126,63,0,383,387,3,50,25,0,384,386,3,54,27,0,385,384,1,0, + 0,0,386,389,1,0,0,0,387,385,1,0,0,0,387,388,1,0,0,0,388,429,1,0,0,0,389, + 387,1,0,0,0,390,392,5,64,0,0,391,393,5,69,0,0,392,391,1,0,0,0,392,393, + 1,0,0,0,393,394,1,0,0,0,394,395,3,126,63,0,395,396,5,83,0,0,396,397,5, + 84,0,0,397,398,3,112,56,0,398,429,1,0,0,0,399,401,5,64,0,0,400,402,5,69, + 0,0,401,400,1,0,0,0,401,402,1,0,0,0,402,403,1,0,0,0,403,404,3,126,63,0, + 404,405,5,68,0,0,405,406,5,84,0,0,406,429,1,0,0,0,407,408,5,68,0,0,408, + 409,5,69,0,0,409,429,3,126,63,0,410,411,5,68,0,0,411,412,5,75,0,0,412, + 429,3,126,63,0,413,414,5,65,0,0,414,415,5,66,0,0,415,429,3,126,63,0,416, + 417,5,65,0,0,417,418,5,69,0,0,418,419,3,126,63,0,419,420,5,66,0,0,420, + 421,3,126,63,0,421,429,1,0,0,0,422,423,5,65,0,0,423,424,5,90,0,0,424,425, + 3,126,63,0,425,426,5,66,0,0,426,427,3,126,63,0,427,429,1,0,0,0,428,371, + 1,0,0,0,428,376,1,0,0,0,428,378,1,0,0,0,428,390,1,0,0,0,428,399,1,0,0, + 0,428,407,1,0,0,0,428,410,1,0,0,0,428,413,1,0,0,0,428,416,1,0,0,0,428, + 422,1,0,0,0,429,35,1,0,0,0,430,434,3,70,35,0,431,434,3,64,32,0,432,434, + 3,20,10,0,433,430,1,0,0,0,433,431,1,0,0,0,433,432,1,0,0,0,434,37,1,0,0, + 0,435,437,5,57,0,0,436,438,5,86,0,0,437,436,1,0,0,0,437,438,1,0,0,0,438, + 439,1,0,0,0,439,440,5,90,0,0,440,441,3,126,63,0,441,442,5,25,0,0,442,443, + 3,126,63,0,443,444,5,135,0,0,444,449,3,42,21,0,445,446,5,137,0,0,446,448, + 3,42,21,0,447,445,1,0,0,0,448,451,1,0,0,0,449,447,1,0,0,0,449,450,1,0, + 0,0,450,452,1,0,0,0,451,449,1,0,0,0,452,455,5,136,0,0,453,454,5,92,0,0, + 454,456,3,44,22,0,455,453,1,0,0,0,455,456,1,0,0,0,456,39,1,0,0,0,457,458, + 5,68,0,0,458,459,5,58,0,0,459,473,3,126,63,0,460,461,5,68,0,0,461,462, + 5,90,0,0,462,463,3,126,63,0,463,464,5,25,0,0,464,465,3,126,63,0,465,473, + 1,0,0,0,466,467,5,68,0,0,467,468,5,99,0,0,468,473,3,126,63,0,469,470,5, + 68,0,0,470,471,5,98,0,0,471,473,3,126,63,0,472,457,1,0,0,0,472,460,1,0, + 0,0,472,466,1,0,0,0,472,469,1,0,0,0,473,41,1,0,0,0,474,476,3,126,63,0, + 475,477,7,1,0,0,476,475,1,0,0,0,476,477,1,0,0,0,477,43,1,0,0,0,478,484, + 5,73,0,0,479,480,5,95,0,0,480,484,5,107,0,0,481,482,5,96,0,0,482,484,5, + 107,0,0,483,478,1,0,0,0,483,479,1,0,0,0,483,481,1,0,0,0,484,45,1,0,0,0, + 485,486,3,126,63,0,486,488,3,50,25,0,487,489,3,48,24,0,488,487,1,0,0,0, + 488,489,1,0,0,0,489,493,1,0,0,0,490,492,3,54,27,0,491,490,1,0,0,0,492, + 495,1,0,0,0,493,491,1,0,0,0,493,494,1,0,0,0,494,47,1,0,0,0,495,493,1,0, + 0,0,496,497,5,5,0,0,497,498,5,135,0,0,498,499,3,112,56,0,499,500,5,136, + 0,0,500,49,1,0,0,0,501,504,3,126,63,0,502,504,5,88,0,0,503,501,1,0,0,0, + 503,502,1,0,0,0,504,506,1,0,0,0,505,507,3,126,63,0,506,505,1,0,0,0,506, + 507,1,0,0,0,507,509,1,0,0,0,508,510,3,126,63,0,509,508,1,0,0,0,509,510, + 1,0,0,0,510,519,1,0,0,0,511,512,5,135,0,0,512,515,3,52,26,0,513,514,5, + 137,0,0,514,516,3,52,26,0,515,513,1,0,0,0,515,516,1,0,0,0,516,517,1,0, + 0,0,517,518,5,136,0,0,518,520,1,0,0,0,519,511,1,0,0,0,519,520,1,0,0,0, + 520,51,1,0,0,0,521,523,5,127,0,0,522,521,1,0,0,0,522,523,1,0,0,0,523,524, + 1,0,0,0,524,525,5,143,0,0,525,53,1,0,0,0,526,527,5,8,0,0,527,596,5,107, + 0,0,528,596,5,107,0,0,529,530,5,84,0,0,530,596,3,112,56,0,531,532,5,92, + 0,0,532,596,7,2,0,0,533,534,5,75,0,0,534,536,3,126,63,0,535,533,1,0,0, + 0,535,536,1,0,0,0,536,537,1,0,0,0,537,538,5,97,0,0,538,539,5,135,0,0,539, + 540,3,58,29,0,540,541,5,136,0,0,541,596,1,0,0,0,542,543,5,75,0,0,543,545, + 3,126,63,0,544,542,1,0,0,0,544,545,1,0,0,0,545,546,1,0,0,0,546,547,5,73, + 0,0,547,549,5,74,0,0,548,550,3,136,68,0,549,548,1,0,0,0,549,550,1,0,0, + 0,550,596,1,0,0,0,551,552,5,75,0,0,552,554,3,126,63,0,553,551,1,0,0,0, + 553,554,1,0,0,0,554,555,1,0,0,0,555,557,5,86,0,0,556,558,3,136,68,0,557, + 556,1,0,0,0,557,558,1,0,0,0,558,596,1,0,0,0,559,560,5,75,0,0,560,562,3, + 126,63,0,561,559,1,0,0,0,561,562,1,0,0,0,562,563,1,0,0,0,563,564,5,77, + 0,0,564,576,3,126,63,0,565,566,5,135,0,0,566,571,3,126,63,0,567,568,5, + 137,0,0,568,570,3,126,63,0,569,567,1,0,0,0,570,573,1,0,0,0,571,569,1,0, + 0,0,571,572,1,0,0,0,572,574,1,0,0,0,573,571,1,0,0,0,574,575,5,136,0,0, + 575,577,1,0,0,0,576,565,1,0,0,0,576,577,1,0,0,0,577,581,1,0,0,0,578,580, + 3,60,30,0,579,578,1,0,0,0,580,583,1,0,0,0,581,579,1,0,0,0,581,582,1,0, + 0,0,582,596,1,0,0,0,583,581,1,0,0,0,584,593,5,88,0,0,585,586,5,135,0,0, + 586,589,3,52,26,0,587,588,5,137,0,0,588,590,3,52,26,0,589,587,1,0,0,0, + 589,590,1,0,0,0,590,591,1,0,0,0,591,592,5,136,0,0,592,594,1,0,0,0,593, + 585,1,0,0,0,593,594,1,0,0,0,594,596,1,0,0,0,595,526,1,0,0,0,595,528,1, + 0,0,0,595,529,1,0,0,0,595,531,1,0,0,0,595,535,1,0,0,0,595,544,1,0,0,0, + 595,553,1,0,0,0,595,561,1,0,0,0,595,584,1,0,0,0,596,55,1,0,0,0,597,598, + 5,75,0,0,598,600,3,126,63,0,599,597,1,0,0,0,599,600,1,0,0,0,600,601,1, + 0,0,0,601,602,5,73,0,0,602,604,5,74,0,0,603,605,3,136,68,0,604,603,1,0, + 0,0,604,605,1,0,0,0,605,606,1,0,0,0,606,607,5,135,0,0,607,612,3,126,63, + 0,608,609,5,137,0,0,609,611,3,126,63,0,610,608,1,0,0,0,611,614,1,0,0,0, + 612,610,1,0,0,0,612,613,1,0,0,0,613,615,1,0,0,0,614,612,1,0,0,0,615,616, + 5,136,0,0,616,687,1,0,0,0,617,618,5,75,0,0,618,620,3,126,63,0,619,617, + 1,0,0,0,619,620,1,0,0,0,620,621,1,0,0,0,621,623,5,86,0,0,622,624,3,136, + 68,0,623,622,1,0,0,0,623,624,1,0,0,0,624,625,1,0,0,0,625,626,5,135,0,0, + 626,631,3,126,63,0,627,628,5,137,0,0,628,630,3,126,63,0,629,627,1,0,0, + 0,630,633,1,0,0,0,631,629,1,0,0,0,631,632,1,0,0,0,632,634,1,0,0,0,633, + 631,1,0,0,0,634,635,5,136,0,0,635,687,1,0,0,0,636,637,5,75,0,0,637,639, + 3,126,63,0,638,636,1,0,0,0,638,639,1,0,0,0,639,640,1,0,0,0,640,641,5,76, + 0,0,641,644,5,74,0,0,642,643,5,85,0,0,643,645,5,90,0,0,644,642,1,0,0,0, + 644,645,1,0,0,0,645,646,1,0,0,0,646,647,5,135,0,0,647,652,3,126,63,0,648, + 649,5,137,0,0,649,651,3,126,63,0,650,648,1,0,0,0,651,654,1,0,0,0,652,650, + 1,0,0,0,652,653,1,0,0,0,653,655,1,0,0,0,654,652,1,0,0,0,655,656,5,136, + 0,0,656,657,5,77,0,0,657,669,3,126,63,0,658,659,5,135,0,0,659,664,3,126, + 63,0,660,661,5,137,0,0,661,663,3,126,63,0,662,660,1,0,0,0,663,666,1,0, + 0,0,664,662,1,0,0,0,664,665,1,0,0,0,665,667,1,0,0,0,666,664,1,0,0,0,667, + 668,5,136,0,0,668,670,1,0,0,0,669,658,1,0,0,0,669,670,1,0,0,0,670,674, + 1,0,0,0,671,673,3,60,30,0,672,671,1,0,0,0,673,676,1,0,0,0,674,672,1,0, + 0,0,674,675,1,0,0,0,675,687,1,0,0,0,676,674,1,0,0,0,677,678,5,75,0,0,678, + 680,3,126,63,0,679,677,1,0,0,0,679,680,1,0,0,0,680,681,1,0,0,0,681,682, + 5,97,0,0,682,683,5,135,0,0,683,684,3,58,29,0,684,685,5,136,0,0,685,687, + 1,0,0,0,686,599,1,0,0,0,686,619,1,0,0,0,686,638,1,0,0,0,686,679,1,0,0, + 0,687,57,1,0,0,0,688,694,8,3,0,0,689,690,5,135,0,0,690,691,3,58,29,0,691, + 692,5,136,0,0,692,694,1,0,0,0,693,688,1,0,0,0,693,689,1,0,0,0,694,697, + 1,0,0,0,695,693,1,0,0,0,695,696,1,0,0,0,696,59,1,0,0,0,697,695,1,0,0,0, + 698,699,5,25,0,0,699,700,5,79,0,0,700,705,3,62,31,0,701,702,5,25,0,0,702, + 703,5,78,0,0,703,705,3,62,31,0,704,698,1,0,0,0,704,701,1,0,0,0,705,61, + 1,0,0,0,706,715,5,80,0,0,707,708,5,85,0,0,708,715,5,82,0,0,709,715,5,81, + 0,0,710,711,5,83,0,0,711,715,5,107,0,0,712,713,5,83,0,0,713,715,5,84,0, + 0,714,706,1,0,0,0,714,707,1,0,0,0,714,709,1,0,0,0,714,710,1,0,0,0,714, + 712,1,0,0,0,715,63,1,0,0,0,716,717,5,70,0,0,717,718,5,71,0,0,718,746,3, + 126,63,0,719,720,5,135,0,0,720,725,3,126,63,0,721,722,5,137,0,0,722,724, + 3,126,63,0,723,721,1,0,0,0,724,727,1,0,0,0,725,723,1,0,0,0,725,726,1,0, + 0,0,726,728,1,0,0,0,727,725,1,0,0,0,728,729,5,136,0,0,729,731,1,0,0,0, + 730,719,1,0,0,0,730,731,1,0,0,0,731,742,1,0,0,0,732,733,5,72,0,0,733,738, + 3,66,33,0,734,735,5,137,0,0,735,737,3,66,33,0,736,734,1,0,0,0,737,740, + 1,0,0,0,738,736,1,0,0,0,738,739,1,0,0,0,739,743,1,0,0,0,740,738,1,0,0, + 0,741,743,3,70,35,0,742,732,1,0,0,0,742,741,1,0,0,0,743,747,1,0,0,0,744, + 745,5,84,0,0,745,747,5,72,0,0,746,730,1,0,0,0,746,744,1,0,0,0,747,65,1, + 0,0,0,748,749,5,135,0,0,749,754,3,68,34,0,750,751,5,137,0,0,751,753,3, + 68,34,0,752,750,1,0,0,0,753,756,1,0,0,0,754,752,1,0,0,0,754,755,1,0,0, + 0,755,757,1,0,0,0,756,754,1,0,0,0,757,758,5,136,0,0,758,67,1,0,0,0,759, + 762,5,84,0,0,760,762,3,112,56,0,761,759,1,0,0,0,761,760,1,0,0,0,762,69, + 1,0,0,0,763,769,3,72,36,0,764,765,3,74,37,0,765,766,3,72,36,0,766,768, + 1,0,0,0,767,764,1,0,0,0,768,771,1,0,0,0,769,767,1,0,0,0,769,770,1,0,0, + 0,770,773,1,0,0,0,771,769,1,0,0,0,772,774,3,108,54,0,773,772,1,0,0,0,773, + 774,1,0,0,0,774,776,1,0,0,0,775,777,3,88,44,0,776,775,1,0,0,0,776,777, + 1,0,0,0,777,71,1,0,0,0,778,793,3,76,38,0,779,780,5,135,0,0,780,781,3,70, + 35,0,781,782,5,136,0,0,782,793,1,0,0,0,783,784,5,72,0,0,784,789,3,66,33, + 0,785,786,5,137,0,0,786,788,3,66,33,0,787,785,1,0,0,0,788,791,1,0,0,0, + 789,787,1,0,0,0,789,790,1,0,0,0,790,793,1,0,0,0,791,789,1,0,0,0,792,778, + 1,0,0,0,792,779,1,0,0,0,792,783,1,0,0,0,793,73,1,0,0,0,794,796,5,53,0, + 0,795,797,5,54,0,0,796,795,1,0,0,0,796,797,1,0,0,0,797,801,1,0,0,0,798, + 801,5,55,0,0,799,801,5,56,0,0,800,794,1,0,0,0,800,798,1,0,0,0,800,799, + 1,0,0,0,801,75,1,0,0,0,802,804,5,1,0,0,803,805,3,78,39,0,804,803,1,0,0, + 0,804,805,1,0,0,0,805,807,1,0,0,0,806,808,3,84,42,0,807,806,1,0,0,0,807, + 808,1,0,0,0,808,809,1,0,0,0,809,812,3,92,46,0,810,811,5,71,0,0,811,813, + 3,126,63,0,812,810,1,0,0,0,812,813,1,0,0,0,813,815,1,0,0,0,814,816,3,96, + 48,0,815,814,1,0,0,0,815,816,1,0,0,0,816,818,1,0,0,0,817,819,3,106,53, + 0,818,817,1,0,0,0,818,819,1,0,0,0,819,821,1,0,0,0,820,822,3,80,40,0,821, + 820,1,0,0,0,821,822,1,0,0,0,822,824,1,0,0,0,823,825,3,82,41,0,824,823, + 1,0,0,0,824,825,1,0,0,0,825,77,1,0,0,0,826,827,7,4,0,0,827,79,1,0,0,0, + 828,829,5,27,0,0,829,830,5,29,0,0,830,835,3,112,56,0,831,832,5,137,0,0, + 832,834,3,112,56,0,833,831,1,0,0,0,834,837,1,0,0,0,835,833,1,0,0,0,835, + 836,1,0,0,0,836,81,1,0,0,0,837,835,1,0,0,0,838,839,5,30,0,0,839,840,3, + 112,56,0,840,83,1,0,0,0,841,842,5,4,0,0,842,847,3,86,43,0,843,844,7,5, + 0,0,844,846,3,86,43,0,845,843,1,0,0,0,846,849,1,0,0,0,847,845,1,0,0,0, + 847,848,1,0,0,0,848,851,1,0,0,0,849,847,1,0,0,0,850,852,5,36,0,0,851,850, + 1,0,0,0,851,852,1,0,0,0,852,85,1,0,0,0,853,860,5,143,0,0,854,860,5,141, + 0,0,855,856,5,135,0,0,856,857,3,112,56,0,857,858,5,136,0,0,858,860,1,0, + 0,0,859,853,1,0,0,0,859,854,1,0,0,0,859,855,1,0,0,0,860,87,1,0,0,0,861, + 862,5,45,0,0,862,863,3,112,56,0,863,870,3,90,45,0,864,865,5,46,0,0,865, + 866,7,6,0,0,866,867,3,112,56,0,867,868,3,90,45,0,868,869,5,51,0,0,869, + 871,1,0,0,0,870,864,1,0,0,0,870,871,1,0,0,0,871,879,1,0,0,0,872,873,5, + 46,0,0,873,874,7,6,0,0,874,875,3,112,56,0,875,876,3,90,45,0,876,877,5, + 51,0,0,877,879,1,0,0,0,878,861,1,0,0,0,878,872,1,0,0,0,879,89,1,0,0,0, + 880,881,7,7,0,0,881,91,1,0,0,0,882,892,5,122,0,0,883,888,3,94,47,0,884, + 885,5,137,0,0,885,887,3,94,47,0,886,884,1,0,0,0,887,890,1,0,0,0,888,886, + 1,0,0,0,888,889,1,0,0,0,889,892,1,0,0,0,890,888,1,0,0,0,891,882,1,0,0, + 0,891,883,1,0,0,0,892,93,1,0,0,0,893,894,3,126,63,0,894,895,5,138,0,0, + 895,896,5,122,0,0,896,905,1,0,0,0,897,902,3,112,56,0,898,900,5,5,0,0,899, + 898,1,0,0,0,899,900,1,0,0,0,900,901,1,0,0,0,901,903,3,126,63,0,902,899, + 1,0,0,0,902,903,1,0,0,0,903,905,1,0,0,0,904,893,1,0,0,0,904,897,1,0,0, + 0,905,95,1,0,0,0,906,907,5,2,0,0,907,912,3,98,49,0,908,909,5,137,0,0,909, + 911,3,98,49,0,910,908,1,0,0,0,911,914,1,0,0,0,912,910,1,0,0,0,912,913, + 1,0,0,0,913,97,1,0,0,0,914,912,1,0,0,0,915,919,3,100,50,0,916,918,3,102, + 51,0,917,916,1,0,0,0,918,921,1,0,0,0,919,917,1,0,0,0,919,920,1,0,0,0,920, + 99,1,0,0,0,921,919,1,0,0,0,922,927,3,126,63,0,923,925,5,5,0,0,924,923, + 1,0,0,0,924,925,1,0,0,0,925,926,1,0,0,0,926,928,3,126,63,0,927,924,1,0, + 0,0,927,928,1,0,0,0,928,943,1,0,0,0,929,930,5,135,0,0,930,931,3,70,35, + 0,931,936,5,136,0,0,932,934,5,5,0,0,933,932,1,0,0,0,933,934,1,0,0,0,934, + 935,1,0,0,0,935,937,3,126,63,0,936,933,1,0,0,0,936,937,1,0,0,0,937,943, + 1,0,0,0,938,939,5,135,0,0,939,940,3,98,49,0,940,941,5,136,0,0,941,943, + 1,0,0,0,942,922,1,0,0,0,942,929,1,0,0,0,942,938,1,0,0,0,943,101,1,0,0, + 0,944,945,3,104,52,0,945,946,5,23,0,0,946,947,3,100,50,0,947,948,5,25, + 0,0,948,949,3,112,56,0,949,960,1,0,0,0,950,951,5,37,0,0,951,952,5,23,0, + 0,952,960,3,100,50,0,953,954,5,37,0,0,954,955,5,38,0,0,955,960,3,100,50, + 0,956,957,5,22,0,0,957,958,5,38,0,0,958,960,3,100,50,0,959,944,1,0,0,0, + 959,950,1,0,0,0,959,953,1,0,0,0,959,956,1,0,0,0,960,103,1,0,0,0,961,963, + 5,18,0,0,962,961,1,0,0,0,962,963,1,0,0,0,963,977,1,0,0,0,964,966,5,19, + 0,0,965,967,5,22,0,0,966,965,1,0,0,0,966,967,1,0,0,0,967,977,1,0,0,0,968, + 970,5,20,0,0,969,971,5,22,0,0,970,969,1,0,0,0,970,971,1,0,0,0,971,977, + 1,0,0,0,972,974,5,21,0,0,973,975,5,22,0,0,974,973,1,0,0,0,974,975,1,0, + 0,0,975,977,1,0,0,0,976,962,1,0,0,0,976,964,1,0,0,0,976,968,1,0,0,0,976, + 972,1,0,0,0,977,105,1,0,0,0,978,979,5,3,0,0,979,980,3,112,56,0,980,107, + 1,0,0,0,981,982,5,26,0,0,982,983,5,29,0,0,983,988,3,110,55,0,984,985,5, + 137,0,0,985,987,3,110,55,0,986,984,1,0,0,0,987,990,1,0,0,0,988,986,1,0, + 0,0,988,989,1,0,0,0,989,109,1,0,0,0,990,988,1,0,0,0,991,993,3,112,56,0, + 992,994,7,1,0,0,993,992,1,0,0,0,993,994,1,0,0,0,994,111,1,0,0,0,995,996, + 6,56,-1,0,996,997,7,5,0,0,997,1002,3,112,56,19,998,999,7,8,0,0,999,1002, + 3,112,56,7,1000,1002,3,114,57,0,1001,995,1,0,0,0,1001,998,1,0,0,0,1001, + 1000,1,0,0,0,1002,1087,1,0,0,0,1003,1004,10,20,0,0,1004,1005,5,125,0,0, + 1005,1086,3,112,56,21,1006,1007,10,18,0,0,1007,1008,7,9,0,0,1008,1086, + 3,112,56,19,1009,1010,10,17,0,0,1010,1011,5,124,0,0,1011,1086,3,112,56, + 18,1012,1013,10,16,0,0,1013,1014,5,17,0,0,1014,1086,3,112,56,17,1015,1016, + 10,15,0,0,1016,1017,7,5,0,0,1017,1086,3,112,56,16,1018,1019,10,14,0,0, + 1019,1020,5,128,0,0,1020,1086,3,112,56,15,1021,1022,10,13,0,0,1022,1023, + 7,10,0,0,1023,1086,3,112,56,14,1024,1026,10,12,0,0,1025,1027,5,8,0,0,1026, + 1025,1,0,0,0,1026,1027,1,0,0,0,1027,1028,1,0,0,0,1028,1029,5,52,0,0,1029, + 1030,3,112,56,0,1030,1031,5,6,0,0,1031,1032,3,112,56,13,1032,1086,1,0, + 0,0,1033,1035,10,11,0,0,1034,1036,5,8,0,0,1035,1034,1,0,0,0,1035,1036, + 1,0,0,0,1036,1037,1,0,0,0,1037,1038,5,16,0,0,1038,1086,3,112,56,12,1039, + 1040,10,6,0,0,1040,1041,7,11,0,0,1041,1086,3,112,56,7,1042,1043,10,5,0, + 0,1043,1044,7,12,0,0,1044,1086,3,112,56,6,1045,1046,10,4,0,0,1046,1047, + 7,13,0,0,1047,1086,3,112,56,5,1048,1049,10,3,0,0,1049,1050,5,10,0,0,1050, + 1086,3,112,56,4,1051,1052,10,2,0,0,1052,1053,5,11,0,0,1053,1086,3,112, + 56,3,1054,1056,10,10,0,0,1055,1057,5,8,0,0,1056,1055,1,0,0,0,1056,1057, + 1,0,0,0,1057,1058,1,0,0,0,1058,1059,5,24,0,0,1059,1060,5,135,0,0,1060, + 1061,3,70,35,0,1061,1062,5,136,0,0,1062,1086,1,0,0,0,1063,1065,10,9,0, + 0,1064,1066,5,8,0,0,1065,1064,1,0,0,0,1065,1066,1,0,0,0,1066,1067,1,0, + 0,0,1067,1068,5,24,0,0,1068,1069,5,135,0,0,1069,1074,3,112,56,0,1070,1071, + 5,137,0,0,1071,1073,3,112,56,0,1072,1070,1,0,0,0,1073,1076,1,0,0,0,1074, + 1072,1,0,0,0,1074,1075,1,0,0,0,1075,1077,1,0,0,0,1076,1074,1,0,0,0,1077, + 1078,5,136,0,0,1078,1086,1,0,0,0,1079,1080,10,8,0,0,1080,1082,5,28,0,0, + 1081,1083,5,8,0,0,1082,1081,1,0,0,0,1082,1083,1,0,0,0,1083,1084,1,0,0, + 0,1084,1086,5,107,0,0,1085,1003,1,0,0,0,1085,1006,1,0,0,0,1085,1009,1, + 0,0,0,1085,1012,1,0,0,0,1085,1015,1,0,0,0,1085,1018,1,0,0,0,1085,1021, + 1,0,0,0,1085,1024,1,0,0,0,1085,1033,1,0,0,0,1085,1039,1,0,0,0,1085,1042, + 1,0,0,0,1085,1045,1,0,0,0,1085,1048,1,0,0,0,1085,1051,1,0,0,0,1085,1054, + 1,0,0,0,1085,1063,1,0,0,0,1085,1079,1,0,0,0,1086,1089,1,0,0,0,1087,1085, + 1,0,0,0,1087,1088,1,0,0,0,1088,113,1,0,0,0,1089,1087,1,0,0,0,1090,1110, + 3,128,64,0,1091,1110,3,116,58,0,1092,1110,3,120,60,0,1093,1110,3,124,62, + 0,1094,1110,5,141,0,0,1095,1110,5,140,0,0,1096,1097,5,31,0,0,1097,1098, + 5,135,0,0,1098,1099,3,70,35,0,1099,1100,5,136,0,0,1100,1110,1,0,0,0,1101, + 1102,5,135,0,0,1102,1103,3,70,35,0,1103,1104,5,136,0,0,1104,1110,1,0,0, + 0,1105,1106,5,135,0,0,1106,1107,3,112,56,0,1107,1108,5,136,0,0,1108,1110, + 1,0,0,0,1109,1090,1,0,0,0,1109,1091,1,0,0,0,1109,1092,1,0,0,0,1109,1093, + 1,0,0,0,1109,1094,1,0,0,0,1109,1095,1,0,0,0,1109,1096,1,0,0,0,1109,1101, + 1,0,0,0,1109,1105,1,0,0,0,1110,115,1,0,0,0,1111,1113,5,41,0,0,1112,1114, + 3,112,56,0,1113,1112,1,0,0,0,1113,1114,1,0,0,0,1114,1116,1,0,0,0,1115, + 1117,3,118,59,0,1116,1115,1,0,0,0,1117,1118,1,0,0,0,1118,1116,1,0,0,0, + 1118,1119,1,0,0,0,1119,1122,1,0,0,0,1120,1121,5,43,0,0,1121,1123,3,112, + 56,0,1122,1120,1,0,0,0,1122,1123,1,0,0,0,1123,1124,1,0,0,0,1124,1125,5, + 44,0,0,1125,117,1,0,0,0,1126,1127,5,42,0,0,1127,1128,3,112,56,0,1128,1129, + 5,33,0,0,1129,1130,3,112,56,0,1130,119,1,0,0,0,1131,1132,3,122,61,0,1132, + 1145,5,135,0,0,1133,1146,5,122,0,0,1134,1136,5,35,0,0,1135,1134,1,0,0, + 0,1135,1136,1,0,0,0,1136,1137,1,0,0,0,1137,1142,3,112,56,0,1138,1139,5, + 137,0,0,1139,1141,3,112,56,0,1140,1138,1,0,0,0,1141,1144,1,0,0,0,1142, + 1140,1,0,0,0,1142,1143,1,0,0,0,1143,1146,1,0,0,0,1144,1142,1,0,0,0,1145, + 1133,1,0,0,0,1145,1135,1,0,0,0,1145,1146,1,0,0,0,1146,1147,1,0,0,0,1147, + 1149,5,136,0,0,1148,1150,3,148,74,0,1149,1148,1,0,0,0,1149,1150,1,0,0, + 0,1150,1152,1,0,0,0,1151,1153,3,146,73,0,1152,1151,1,0,0,0,1152,1153,1, + 0,0,0,1153,1162,1,0,0,0,1154,1156,3,150,75,0,1155,1154,1,0,0,0,1155,1156, + 1,0,0,0,1156,1158,1,0,0,0,1157,1159,3,152,76,0,1158,1157,1,0,0,0,1158, + 1159,1,0,0,0,1159,1160,1,0,0,0,1160,1161,5,39,0,0,1161,1163,3,134,67,0, + 1162,1155,1,0,0,0,1162,1163,1,0,0,0,1163,121,1,0,0,0,1164,1171,3,126,63, + 0,1165,1171,5,19,0,0,1166,1171,5,20,0,0,1167,1171,5,103,0,0,1168,1171, + 5,48,0,0,1169,1171,5,40,0,0,1170,1164,1,0,0,0,1170,1165,1,0,0,0,1170,1166, + 1,0,0,0,1170,1167,1,0,0,0,1170,1168,1,0,0,0,1170,1169,1,0,0,0,1171,123, + 1,0,0,0,1172,1173,3,126,63,0,1173,1174,5,138,0,0,1174,1176,1,0,0,0,1175, + 1172,1,0,0,0,1175,1176,1,0,0,0,1176,1177,1,0,0,0,1177,1178,3,126,63,0, + 1178,125,1,0,0,0,1179,1184,5,150,0,0,1180,1184,5,148,0,0,1181,1184,5,149, + 0,0,1182,1184,3,144,72,0,1183,1179,1,0,0,0,1183,1180,1,0,0,0,1183,1181, + 1,0,0,0,1183,1182,1,0,0,0,1184,127,1,0,0,0,1185,1195,5,143,0,0,1186,1195, + 5,144,0,0,1187,1195,5,142,0,0,1188,1195,5,145,0,0,1189,1195,5,146,0,0, + 1190,1195,5,147,0,0,1191,1195,5,105,0,0,1192,1195,5,106,0,0,1193,1195, + 5,107,0,0,1194,1185,1,0,0,0,1194,1186,1,0,0,0,1194,1187,1,0,0,0,1194,1188, + 1,0,0,0,1194,1189,1,0,0,0,1194,1190,1,0,0,0,1194,1191,1,0,0,0,1194,1192, + 1,0,0,0,1194,1193,1,0,0,0,1195,129,1,0,0,0,1196,1198,5,59,0,0,1197,1199, + 7,14,0,0,1198,1197,1,0,0,0,1198,1199,1,0,0,0,1199,1209,1,0,0,0,1200,1202, + 5,60,0,0,1201,1203,7,14,0,0,1202,1201,1,0,0,0,1202,1203,1,0,0,0,1203,1209, + 1,0,0,0,1204,1206,5,61,0,0,1205,1207,7,14,0,0,1206,1205,1,0,0,0,1206,1207, + 1,0,0,0,1207,1209,1,0,0,0,1208,1196,1,0,0,0,1208,1200,1,0,0,0,1208,1204, + 1,0,0,0,1209,131,1,0,0,0,1210,1211,3,112,56,0,1211,1212,5,0,0,1,1212,133, + 1,0,0,0,1213,1224,5,135,0,0,1214,1215,5,40,0,0,1215,1216,5,29,0,0,1216, + 1221,3,112,56,0,1217,1218,5,137,0,0,1218,1220,3,112,56,0,1219,1217,1,0, + 0,0,1220,1223,1,0,0,0,1221,1219,1,0,0,0,1221,1222,1,0,0,0,1222,1225,1, + 0,0,0,1223,1221,1,0,0,0,1224,1214,1,0,0,0,1224,1225,1,0,0,0,1225,1227, + 1,0,0,0,1226,1228,3,108,54,0,1227,1226,1,0,0,0,1227,1228,1,0,0,0,1228, + 1230,1,0,0,0,1229,1231,3,138,69,0,1230,1229,1,0,0,0,1230,1231,1,0,0,0, + 1231,1232,1,0,0,0,1232,1233,5,136,0,0,1233,135,1,0,0,0,1234,1235,7,15, + 0,0,1235,137,1,0,0,0,1236,1243,7,16,0,0,1237,1238,5,52,0,0,1238,1239,3, + 140,70,0,1239,1240,5,6,0,0,1240,1241,3,140,70,0,1241,1244,1,0,0,0,1242, + 1244,3,140,70,0,1243,1237,1,0,0,0,1243,1242,1,0,0,0,1244,1247,1,0,0,0, + 1245,1246,5,114,0,0,1246,1248,3,142,71,0,1247,1245,1,0,0,0,1247,1248,1, + 0,0,0,1248,139,1,0,0,0,1249,1250,5,110,0,0,1250,1257,7,17,0,0,1251,1252, + 5,113,0,0,1252,1257,5,50,0,0,1253,1254,3,112,56,0,1254,1255,7,17,0,0,1255, + 1257,1,0,0,0,1256,1249,1,0,0,0,1256,1251,1,0,0,0,1256,1253,1,0,0,0,1257, + 141,1,0,0,0,1258,1259,5,113,0,0,1259,1265,5,50,0,0,1260,1265,5,27,0,0, + 1261,1265,5,115,0,0,1262,1263,5,85,0,0,1263,1265,5,116,0,0,1264,1258,1, + 0,0,0,1264,1260,1,0,0,0,1264,1261,1,0,0,0,1264,1262,1,0,0,0,1265,143,1, + 0,0,0,1266,1267,7,18,0,0,1267,145,1,0,0,0,1268,1269,5,121,0,0,1269,1270, + 5,135,0,0,1270,1271,5,3,0,0,1271,1272,3,112,56,0,1272,1273,5,136,0,0,1273, + 147,1,0,0,0,1274,1275,5,117,0,0,1275,1276,5,27,0,0,1276,1277,5,135,0,0, + 1277,1278,3,108,54,0,1278,1279,5,136,0,0,1279,149,1,0,0,0,1280,1281,5, + 2,0,0,1281,1282,7,19,0,0,1282,151,1,0,0,0,1283,1284,7,20,0,0,1284,1285, + 5,120,0,0,1285,153,1,0,0,0,162,155,171,174,180,200,209,212,220,229,233, + 245,253,257,265,270,273,281,288,298,305,319,324,333,344,354,357,364,373, + 380,387,392,401,428,433,437,449,455,472,476,483,488,493,503,506,509,515, + 519,522,535,544,549,553,557,561,571,576,581,589,593,595,599,604,612,619, + 623,631,638,644,652,664,669,674,679,686,693,695,704,714,725,730,738,742, + 746,754,761,769,773,776,789,792,796,800,804,807,812,815,818,821,824,835, + 847,851,859,870,878,888,891,899,902,904,912,919,924,927,933,936,942,959, + 962,966,970,974,976,988,993,1001,1026,1035,1056,1065,1074,1082,1085,1087, + 1109,1113,1118,1122,1135,1142,1145,1149,1152,1155,1158,1162,1170,1175, + 1183,1194,1198,1202,1206,1208,1221,1224,1227,1230,1243,1247,1256,1264 }; public static readonly ATN _ATN = diff --git a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlVisitor.cs b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlVisitor.cs index fc4d7ae71..366aaa7d5 100644 --- a/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlVisitor.cs +++ b/src/LibRed/LibRed.Sql/Grammar/Generated/AccessSqlVisitor.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -// Generated from AccessSql.g4 by ANTLR 4.13.1 +// Generated from D:/toolkits/efcorejetlibred/src/LibRed/LibRed.Sql/Grammar/AccessSql.g4 by ANTLR 4.13.1 // Unreachable code detected #pragma warning disable 0162 @@ -352,6 +352,13 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitColumnReferencesConstraint([NotNull] AccessSqlParser.ColumnReferencesConstraintContext context); /// + /// Visit a parse tree produced by the IdentityConstraint + /// labeled alternative in . + /// + /// The parse tree. + /// The visitor result. + Result VisitIdentityConstraint([NotNull] AccessSqlParser.IdentityConstraintContext context); + /// /// Visit a parse tree produced by the PrimaryKeyTableConstraint /// labeled alternative in . /// @@ -668,6 +675,13 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitAndExpr([NotNull] AccessSqlParser.AndExprContext context); /// + /// Visit a parse tree produced by the IntDivExpr + /// labeled alternative in . + /// + /// The parse tree. + /// The visitor result. + Result VisitIntDivExpr([NotNull] AccessSqlParser.IntDivExprContext context); + /// /// Visit a parse tree produced by the LikeExpr /// labeled alternative in . /// @@ -689,19 +703,19 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitComparisonExpr([NotNull] AccessSqlParser.ComparisonExprContext context); /// - /// Visit a parse tree produced by the BetweenExpr + /// Visit a parse tree produced by the XorExpr /// labeled alternative in . /// /// The parse tree. /// The visitor result. - Result VisitBetweenExpr([NotNull] AccessSqlParser.BetweenExprContext context); + Result VisitXorExpr([NotNull] AccessSqlParser.XorExprContext context); /// - /// Visit a parse tree produced by the BitNotExpr + /// Visit a parse tree produced by the BetweenExpr /// labeled alternative in . /// /// The parse tree. /// The visitor result. - Result VisitBitNotExpr([NotNull] AccessSqlParser.BitNotExprContext context); + Result VisitBetweenExpr([NotNull] AccessSqlParser.BetweenExprContext context); /// /// Visit a parse tree produced by the PowExpr /// labeled alternative in . @@ -731,6 +745,13 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitInSubqueryExpr([NotNull] AccessSqlParser.InSubqueryExprContext context); /// + /// Visit a parse tree produced by the ConcatExpr + /// labeled alternative in . + /// + /// The parse tree. + /// The visitor result. + Result VisitConcatExpr([NotNull] AccessSqlParser.ConcatExprContext context); + /// /// Visit a parse tree produced by the MulDivExpr /// labeled alternative in . /// @@ -738,12 +759,12 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitMulDivExpr([NotNull] AccessSqlParser.MulDivExprContext context); /// - /// Visit a parse tree produced by the BitwiseExpr + /// Visit a parse tree produced by the ImpExpr /// labeled alternative in . /// /// The parse tree. /// The visitor result. - Result VisitBitwiseExpr([NotNull] AccessSqlParser.BitwiseExprContext context); + Result VisitImpExpr([NotNull] AccessSqlParser.ImpExprContext context); /// /// Visit a parse tree produced by the PrimaryExpr /// labeled alternative in . @@ -759,6 +780,13 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitNotExpr([NotNull] AccessSqlParser.NotExprContext context); /// + /// Visit a parse tree produced by the ModExpr + /// labeled alternative in . + /// + /// The parse tree. + /// The visitor result. + Result VisitModExpr([NotNull] AccessSqlParser.ModExprContext context); + /// /// Visit a parse tree produced by the InExpr /// labeled alternative in . /// @@ -766,12 +794,19 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The visitor result. Result VisitInExpr([NotNull] AccessSqlParser.InExprContext context); /// - /// Visit a parse tree produced by the AddConcatExpr + /// Visit a parse tree produced by the AddSubExpr + /// labeled alternative in . + /// + /// The parse tree. + /// The visitor result. + Result VisitAddSubExpr([NotNull] AccessSqlParser.AddSubExprContext context); + /// + /// Visit a parse tree produced by the EqvExpr /// labeled alternative in . /// /// The parse tree. /// The visitor result. - Result VisitAddConcatExpr([NotNull] AccessSqlParser.AddConcatExprContext context); + Result VisitEqvExpr([NotNull] AccessSqlParser.EqvExprContext context); /// /// Visit a parse tree produced by the LiteralPrimary /// labeled alternative in . @@ -967,5 +1002,59 @@ public interface IAccessSqlVisitor : IParseTreeVisitor { /// The parse tree. /// The visitor result. Result VisitWindowSpecification([NotNull] AccessSqlParser.WindowSpecificationContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitClusteredOption([NotNull] AccessSqlParser.ClusteredOptionContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitWindowFrame([NotNull] AccessSqlParser.WindowFrameContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitFrameBound([NotNull] AccessSqlParser.FrameBoundContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitFrameExclusion([NotNull] AccessSqlParser.FrameExclusionContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitNonReservedKeyword([NotNull] AccessSqlParser.NonReservedKeywordContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitFilterClause([NotNull] AccessSqlParser.FilterClauseContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitWithinGroup([NotNull] AccessSqlParser.WithinGroupContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitNthRowFrom([NotNull] AccessSqlParser.NthRowFromContext context); + /// + /// Visit a parse tree produced by . + /// + /// The parse tree. + /// The visitor result. + Result VisitNullTreatment([NotNull] AccessSqlParser.NullTreatmentContext context); } } // namespace LibRed.Sql.Grammar diff --git a/src/LibRed/LibRed.Sql/Parsing/AstBuilder.cs b/src/LibRed/LibRed.Sql/Parsing/AstBuilder.cs index cc6088039..4891482c0 100644 --- a/src/LibRed/LibRed.Sql/Parsing/AstBuilder.cs +++ b/src/LibRed/LibRed.Sql/Parsing/AstBuilder.cs @@ -85,7 +85,7 @@ private ExecuteStatement BuildExecute(ExecuteStatementContext ctx) => private static DeleteStatement BuildDelete(DeleteStatementContext ctx) => new(OptionalIdentifier(ctx.target), - BuildTableSource(ctx.tableSource()), + BuildTableSources(ctx.tableSource()), ctx.whereClause() is { } w ? BuildExpression(w.expression()) : null); private static UpdateStatement BuildUpdate(UpdateStatementContext ctx) @@ -95,7 +95,7 @@ private static UpdateStatement BuildUpdate(UpdateStatementContext ctx) OptionalIdentifier(a.target.qualifier), Identifier(a.target.name), BuildExpression(a.expression()))) .ToList(); Expression? where = ctx.whereClause() is { } w ? BuildExpression(w.expression()) : null; - return new UpdateStatement(BuildTableSource(ctx.tableSource()), assignments, where); + return new UpdateStatement(BuildTableSources(ctx.tableSource()), assignments, where); } private static SystemVariableSelectStatement BuildSystemVariableSelect(SystemVariableSelectContext ctx) @@ -122,24 +122,24 @@ private static SqlStatement BuildCreateTable(CreateTableStatementContext ctx) // The PRIMARY KEY constraint's name, from whichever form declared it (column- or table-level). string? primaryKeyName = null; + // Where each foreign key and the primary key sit in the statement's text, for a self-reference below. + var foreignKeyTokens = new List(); + int primaryKeyToken = int.MaxValue; + // Column-level UNIQUE and REFERENCES (the single-field forms) apply to the column they follow. foreach (ColumnDefinitionContext cd in ctx.columnDefinition()) { - string columnName = Identifier(cd.name); - foreach (ColumnConstraintContext cc in cd.columnConstraint()) + ColumnKeys keys = ColumnKeysOf(cd); + foreach (PrimaryKeyConstraintContext p in keys.PrimaryKeys) + { + if (p.cname is not null) primaryKeyName = Identifier(p.cname); + primaryKeyToken = Math.Min(primaryKeyToken, p.Start.TokenIndex); + } + uniques.AddRange(keys.Uniques); + foreach ((ForeignKeyConstraint fk, int token) in keys.References) { - switch (cc) - { - case PrimaryKeyConstraintContext p when p.cname is not null: - primaryKeyName = Identifier(p.cname); - break; - case UniqueColumnConstraintContext u: - uniques.Add(new UniqueConstraint(u.cname is null ? null : Identifier(u.cname), [columnName])); - break; - case ColumnReferencesConstraintContext r: - foreignKeys.Add(BuildColumnReferences(r, columnName)); - break; - } + foreignKeys.Add(fk); + foreignKeyTokens.Add(token); } } @@ -150,12 +150,14 @@ private static SqlStatement BuildCreateTable(CreateTableStatementContext ctx) case PrimaryKeyTableConstraintContext pk: primaryKey.AddRange(pk._columns.Select(Identifier)); if (pk.name is not null) primaryKeyName = Identifier(pk.name); + primaryKeyToken = Math.Min(primaryKeyToken, pk.Start.TokenIndex); break; case UniqueTableConstraintContext uq: uniques.Add(new UniqueConstraint(uq.name is null ? null : Identifier(uq.name), uq._columns.Select(Identifier).ToList())); break; case ForeignKeyTableConstraintContext fk: foreignKeys.Add(BuildForeignKey(fk)); + foreignKeyTokens.Add(fk.Start.TokenIndex); break; case CheckTableConstraintContext ck: checks.Add(new CheckConstraint( @@ -164,7 +166,22 @@ private static SqlStatement BuildCreateTable(CreateTableStatementContext ctx) } } - return new CreateTableStatement(Identifier(ctx.table), columns, primaryKey, foreignKeys, uniques, checks, primaryKeyName); + // REFERENCES with no column list names the parent's primary key. For a table referencing itself that is + // the key this statement declares, but only if it is declared earlier in the text: ACE resolves each + // reference as it reaches it, so against a key declared after it there is none yet (verified — ACE then + // reports that the table has no primary key). Resolved here, where the text order is known; a reference + // left without columns is refused when the statement runs. + string table = Identifier(ctx.table); + for (int i = 0; i < foreignKeys.Count; i++) + { + ForeignKeyConstraint fk = foreignKeys[i]; + if (fk.ReferencedColumns.Count == 0 + && string.Equals(fk.ReferencedTable, table, StringComparison.OrdinalIgnoreCase) + && primaryKeyToken < foreignKeyTokens[i]) + foreignKeys[i] = fk with { ReferencedColumns = primaryKey.ToList() }; + } + + return new CreateTableStatement(table, columns, primaryKey, foreignKeys, uniques, checks, primaryKeyName); } /// The verbatim source text of a parse context (preserving spacing), via the input stream — @@ -179,14 +196,15 @@ private static SqlStatement BuildAlterTable(AlterTableStatementContext ctx) { AlterTableAction action = ctx.alterTableAction() switch { - AddColumnActionContext a => new AddColumnAction(BuildColumnDefinition(a.columnDefinition())), + AddColumnActionContext a => BuildAddColumn(a.columnDefinition()), AddConstraintActionContext a => BuildAddConstraint(a.tableConstraint()), AlterColumnActionContext a => new AlterColumnAction( Identifier(a.field), TypeName(a.dataType()), Size(a.dataType()), Scale(a.dataType()), a.columnConstraint().OfType().FirstOrDefault()?.expression().GetText(), // NOT NULL → true, NULL → false, neither → null (leave the column's nullability unchanged). a.columnConstraint().OfType().Any() ? true - : a.columnConstraint().OfType().Any() ? false : null), + : a.columnConstraint().OfType().Any() ? false : null, + IdentityOf(a.columnConstraint())), AlterColumnSetDefaultActionContext a => new AlterColumnSetDefaultAction( Identifier(a.field), OriginalText(a.expression())), AlterColumnDropDefaultActionContext a => new AlterColumnDropDefaultAction(Identifier(a.field)), @@ -200,6 +218,38 @@ private static SqlStatement BuildAlterTable(AlterTableStatementContext ctx) return new AlterTableStatement(Identifier(ctx.table), action); } + /// ADD COLUMN with the constraints its column carries: they apply to the new column, as in CREATE + /// TABLE. + private static AddColumnAction BuildAddColumn(ColumnDefinitionContext ctx) + { + ColumnKeys keys = ColumnKeysOf(ctx); + return new AddColumnAction( + BuildColumnDefinition(ctx), + keys.References.FirstOrDefault().Constraint, + keys.Uniques.FirstOrDefault(), + keys.PrimaryKeys.FirstOrDefault()?.cname is { } name ? Identifier(name) : null); + } + + /// The keys a column's own constraints declare, for CREATE TABLE and ADD COLUMN alike: its PRIMARY KEY + /// constraints, a unique constraint per UNIQUE, and a foreign key per REFERENCES with where each sits in the + /// statement's text. + private readonly record struct ColumnKeys( + List PrimaryKeys, + List Uniques, + List<(ForeignKeyConstraint Constraint, int Token)> References); + + private static ColumnKeys ColumnKeysOf(ColumnDefinitionContext column) + { + string name = Identifier(column.name); + ColumnConstraintContext[] constraints = column.columnConstraint(); + return new ColumnKeys( + constraints.OfType().ToList(), + constraints.OfType() + .Select(u => new UniqueConstraint(u.cname is null ? null : Identifier(u.cname), [name])).ToList(), + constraints.OfType() + .Select(r => (BuildColumnReferences(r, name), r.Start.TokenIndex)).ToList()); + } + private static AlterTableAction BuildAddConstraint(TableConstraintContext tc) => tc switch { PrimaryKeyTableConstraintContext pk => @@ -212,8 +262,11 @@ private static SqlStatement BuildAlterTable(AlterTableStatementContext ctx) _ => throw new SqlParseException("Unsupported ALTER TABLE ADD CONSTRAINT."), }; - private static int? Size(DataTypeContext type) => type.size is { } s ? int.Parse(s.GetText(), CultureInfo.InvariantCulture) : null; - private static int? Scale(DataTypeContext type) => type.scale is { } s ? int.Parse(s.GetText(), CultureInfo.InvariantCulture) : null; + private static int? Size(DataTypeContext type) => SignedInteger(type.size); + private static int? Scale(DataTypeContext type) => SignedInteger(type.scale); + + private static int? SignedInteger(SignedIntegerContext? value) => + value is null ? null : int.Parse(value.GetText(), CultureInfo.InvariantCulture); private static ForeignKeyConstraint BuildForeignKey(ForeignKeyTableConstraintContext ctx) { @@ -273,8 +326,8 @@ private static (ReferentialAction OnUpdate, ReferentialAction OnDelete) ReadFore private static ColumnDefinition BuildColumnDefinition(ColumnDefinitionContext ctx) { DataTypeContext type = ctx.dataType(); - int? size = type.size is { } s ? int.Parse(s.GetText(), CultureInfo.InvariantCulture) : null; - int? scale = type.scale is { } sc ? int.Parse(sc.GetText(), CultureInfo.InvariantCulture) : null; + int? size = Size(type); + int? scale = Scale(type); string typeName = TypeName(type); @@ -291,7 +344,35 @@ private static ColumnDefinition BuildColumnDefinition(ColumnDefinitionContext ct return new ColumnDefinition( Identifier(ctx.name), typeName, size, scale, notNull, primaryKey, defaultSql, compressed, - calculated); + calculated, IdentityOf(ctx.columnConstraint())); + } + + /// A column's IDENTITY attribute — the last one written, when there are several — or null. ACE takes + /// it only straight after the type, NULL/NOT NULL or another IDENTITY: after DEFAULT, PRIMARY KEY or any + /// other constraint it is a syntax error there (verified), so it is one here too. + private static IdentityAttribute? IdentityOf(IEnumerable constraints) + { + IdentityAttribute? identity = null; + bool afterOtherConstraint = false; + foreach (ColumnConstraintContext constraint in constraints) + { + switch (constraint) + { + case IdentityConstraintContext id: + if (afterOtherConstraint) + throw new SqlParseException( + "Syntax error in field definition: IDENTITY must come before DEFAULT, PRIMARY KEY and the " + + "column's other constraints."); + identity = new IdentityAttribute(SignedInteger(id.seed), SignedInteger(id.increment)); + break; + case NotNullConstraintContext or NullableConstraintContext: + break; + default: + afterOtherConstraint = true; + break; + } + } + return identity; } private static SqlStatement BuildCreateIndex(CreateIndexStatementContext ctx) @@ -392,9 +473,9 @@ private static string ParamName(ProcParamContext p) => p.pname.PARAM() is { } at /// The declared type name of a data type — up to three words (e.g. "national character varying") /// joined by single spaces. private static string TypeName(DataTypeContext type) => string.Join(' ', - new[] { type.typeName, type.extra, type.extra2 } - .Where(t => t is not null) - .Select(Identifier)); + new[] { type.identityType is null ? null : "IDENTITY" } + .Concat(new[] { type.typeName, type.extra, type.extra2 }.Where(t => t is not null).Select(Identifier)) + .Where(t => t is not null)); // ---- PARAMETERS-clause lowering: unqualified references to a declared parameter become parameters ---- @@ -444,27 +525,31 @@ private static SelectStatement LowerSelect(SelectStatement sel, HashSet private static Expression LowerExpr(Expression e, HashSet names) => e switch { ColumnReference { Table: null, Column: var c } when names.Contains(c) => new ParameterExpression(c), - BinaryExpression b => b with { Left = LowerExpr(b.Left, names), Right = LowerExpr(b.Right, names) }, - UnaryExpression u => u with { Operand = LowerExpr(u.Operand, names) }, - FunctionCall f => f with { Arguments = f.Arguments.Select(a => LowerExpr(a, names)).ToList() }, // A window function lowers like any other call — arguments AND the OVER clause, since a PARAMETERS name // can appear in a PARTITION BY or ORDER BY expression just as readily as in an argument. WindowFunction w => w with { Arguments = w.Arguments.Select(a => LowerExpr(a, names)).ToList(), + Filter = w.Filter is null ? null : LowerExpr(w.Filter, names), Over = w.Over with { PartitionBy = w.Over.PartitionBy.Select(p => LowerExpr(p, names)).ToList(), OrderBy = w.Over.OrderBy.Select(o => o with { Value = LowerExpr(o.Value, names) }).ToList(), + Frame = w.Over.Frame is { } frame + ? frame with { Start = LowerBound(frame.Start, names), End = LowerBound(frame.End, names) } + : null, }, }, // LowerParameters, not LowerSelect: a subquery may be a set operation or a table value constructor. ScalarSubquery s => new ScalarSubquery(LowerParameters(s.Query, names)), ExistsExpression x => new ExistsExpression(LowerParameters(x.Query, names)), InSubqueryExpression i => i with { Value = LowerExpr(i.Value, names), Query = LowerParameters(i.Query, names) }, - _ => e, + _ => e.MapOperands(o => LowerExpr(o, names)), }; + private static FrameBound LowerBound(FrameBound bound, HashSet names) => + bound.Offset is null ? bound : bound with { Offset = LowerExpr(bound.Offset, names) }; + /// Decomposes a view's "simple SELECT" into the columns/tables/joins/where Access stores as /// MSysQueries rows. Rejects anything Access itself rejects in a view (UNION, GROUP BY/aggregates, /// HAVING, ORDER BY) or that we can't decompose (a derived-table/subquery source). @@ -550,13 +635,13 @@ private static (string Left, string Right) JoinSides(ExpressionContext condition var qualifiers = new List(); void Walk(Expression e) { - switch (e) + if (e is ColumnReference { Table: { } q }) { - case ColumnReference { Table: { } q } when !qualifiers.Contains(q): qualifiers.Add(q); break; - case BinaryExpression b: Walk(b.Left); Walk(b.Right); break; - case UnaryExpression u: Walk(u.Operand); break; - case FunctionCall f: foreach (Expression a in f.Arguments) Walk(a); break; + if (!qualifiers.Contains(q)) qualifiers.Add(q); + return; } + foreach (Expression operand in e.Operands() ?? []) + Walk(operand); } Walk(BuildExpression(condition)); return (qualifiers.ElementAtOrDefault(0) ?? "", qualifiers.ElementAtOrDefault(1) ?? ""); @@ -797,11 +882,13 @@ Expression Operand(TopOperandContext o) => _ => throw new SqlParseException($"Unsupported select item: {ctx.GetText()}"), }; - private static TableReference BuildFrom(FromClauseContext ctx) + private static TableReference BuildFrom(FromClauseContext ctx) => BuildTableSources(ctx.tableSource()); + + /// A comma list of table sources, which is an implicit cross join (no ON). + private static TableReference BuildTableSources(TableSourceContext[] sources) { - TableReference table = BuildTableSource(ctx.tableSource(0)); - // Comma between sources is an implicit cross join (no ON). - foreach (TableSourceContext src in ctx.tableSource().Skip(1)) + TableReference table = BuildTableSource(sources[0]); + foreach (TableSourceContext src in sources.Skip(1)) table = new JoinTable(table, BuildTableSource(src), JoinKind.Cross, null); return table; } @@ -853,12 +940,20 @@ private static OrderByItem BuildOrderByItem(OrderByItemContext ctx) => internal static Expression BuildExpression(ExpressionContext ctx) => ctx switch { - NotExprContext n => new UnaryExpression(UnaryOperator.Not, BuildExpression(n.expression())), - BitNotExprContext n => new UnaryExpression(UnaryOperator.BitNot, BuildExpression(n.expression())), - NegateExprContext n => new UnaryExpression(UnaryOperator.Negate, BuildExpression(n.expression())), - PowExprContext p => new BinaryExpression(BinaryOperator.Power, BuildExpression(p.left), BuildExpression(p.right)), + NotExprContext n => new UnaryExpression( + n.op.Type == BNOT ? UnaryOperator.BitNot : UnaryOperator.Not, BuildExpression(n.expression())), + // A unary plus leaves its operand as it is. + NegateExprContext n => n.op.Type == PLUS + ? BuildExpression(n.expression()) + : SignedNumber(n) is { } signed + ? PowerChain(signed.Number, signed.Exponents) + : new UnaryExpression(UnaryOperator.Negate, BuildExpression(n.expression())), + PowExprContext p => BuildPower(p), MulDivExprContext m => Binary(m.op, m.left, m.right), - AddConcatExprContext a => Binary(a.op, a.left, a.right), + IntDivExprContext d => Binary(d.op, d.left, d.right), + ModExprContext m => Binary(m.op, m.left, m.right), + AddSubExprContext a => Binary(a.op, a.left, a.right), + ConcatExprContext c => Binary(c.op, c.left, c.right), ComparisonExprContext c => Binary(c.op, c.left, c.right), BetweenExprContext b => BuildBetween(b), InExprContext i => BuildIn(i), @@ -868,9 +963,11 @@ private static OrderByItem BuildOrderByItem(OrderByItemContext ctx) => ? new BinaryExpression(BinaryOperator.Like, BuildExpression(l.left), BuildExpression(l.right)) : new UnaryExpression(UnaryOperator.Not, new BinaryExpression(BinaryOperator.Like, BuildExpression(l.left), BuildExpression(l.right))), IsNullExprContext n => new UnaryExpression(n.not is null ? UnaryOperator.IsNull : UnaryOperator.IsNotNull, BuildExpression(n.operand)), - BitwiseExprContext b => Binary(b.op, b.left, b.right), - AndExprContext a => new BinaryExpression(BinaryOperator.And, BuildExpression(a.left), BuildExpression(a.right)), - OrExprContext o => new BinaryExpression(BinaryOperator.Or, BuildExpression(o.left), BuildExpression(o.right)), + AndExprContext a => Binary(a.op, a.left, a.right), + OrExprContext o => Binary(o.op, o.left, o.right), + XorExprContext x => Binary(x.op, x.left, x.right), + EqvExprContext e => Binary(e.op, e.left, e.right), + ImpExprContext i => Binary(i.op, i.left, i.right), PrimaryExprContext p => BuildPrimary(p.primary()), _ => throw new SqlParseException($"Unsupported expression: {ctx.GetText()}"), }; @@ -910,19 +1007,107 @@ private static Expression BuildCase(CaseExpressionContext ctx) private static Expression BuildFunctionCall(FunctionCallContext ctx) { - IReadOnlyList args = ctx.star is not null + string name = FunctionName(ctx.name); + List args = ctx.star is not null ? [new StarExpression()] : ctx.expression().Select(BuildExpression).ToList(); + IReadOnlyList? withinGroup = BuildWithinGroup(ctx, name, args); + Expression? filter = ctx.filterClause() is { } f ? BuildExpression(f.condition) : null; + // An OVER clause turns the same call into a window function, which is a different kind of node rather // than a FunctionCall carrying a spec — see WindowFunction for why the distinction has to be in the type. - return ctx.windowSpecification() is { } over - ? new WindowFunction(FunctionName(ctx.name), args, BuildWindowSpec(over)) - : new FunctionCall(FunctionName(ctx.name), args, Distinct: ctx.distinct is not null); + if (ctx.windowSpecification() is { } over) + { + return new WindowFunction(name, args, BuildWindowSpec(over), + Distinct: ctx.distinct is not null, + IgnoreNulls: ctx.nullTreatment() is { } nulls ? nulls.treatment.Type == IGNORE : null, + FromLast: ctx.nthRowFrom() is { } from ? from.edge.Type == LAST : null, + WithinGroup: withinGroup, + Filter: filter); + } + return new FunctionCall(name, args, Distinct: ctx.distinct is not null, WithinGroup: withinGroup, Filter: filter); + } + + /// + /// An ordered-set aggregate's WITHIN GROUP: its ORDER BY keys, appended to , and their + /// directions. The syntax belongs to those aggregates alone, and they cannot go without it. A percentile takes + /// one argument, the fraction, no DISTINCT and one key; LISTAGG takes the value and optionally a separator, which + /// the standard makes a string literal, and any number of keys. + /// + private static IReadOnlyList? BuildWithinGroup(FunctionCallContext ctx, string name, List args) + { + bool orderedSet = FunctionCall.IsOrderedSetAggregate(name); + if (ctx.withinGroup() is not { } within) + { + return orderedSet + ? throw new SqlParseException($"{name} needs WITHIN GROUP (ORDER BY …).") + : null; + } + if (!orderedSet) + throw new SqlParseException($"{name} takes no WITHIN GROUP."); + if (ctx.star is not null) + throw new SqlParseException($"{name} takes no *."); + + var keys = within.orderByClause().orderByItem().Select(BuildOrderByItem).ToList(); + if (name.Equals("LISTAGG", StringComparison.OrdinalIgnoreCase)) + { + if (args.Count is not (1 or 2) || args is [_, not LiteralExpression { Value: string }]) + throw new SqlParseException("LISTAGG takes a value and, optionally, a separator written as a string."); + } + else if (ctx.distinct is not null || args.Count != 1 || keys.Count != 1) + throw new SqlParseException($"{name} takes one argument, the fraction, no DISTINCT, and orders by one key."); + + args.AddRange(keys.Select(k => k.Value)); + return keys.Select(k => k.Direction).ToList(); } private static WindowSpec BuildWindowSpec(WindowSpecificationContext ctx) => new(ctx._partition.Select(BuildExpression).ToList(), - ctx.orderByClause() is { } o ? o.orderByItem().Select(BuildOrderByItem).ToList() : []); + ctx.orderByClause() is { } o ? o.orderByItem().Select(BuildOrderByItem).ToList() : [], + ctx.windowFrame() is { } frame ? BuildWindowFrame(frame) : null); + + /// + /// A frame clause, held to the standard's syntax rules: the start is not UNBOUNDED FOLLOWING, the end is not + /// UNBOUNDED PRECEDING, and the start does not come after the end in window order — so CURRENT ROW AND 1 + /// PRECEDING is refused, and a lone n FOLLOWING, which ends at the current row, is too. + /// + private static WindowFrame BuildWindowFrame(WindowFrameContext ctx) + { + FrameBound start = BuildFrameBound(ctx.start); + FrameBound end = ctx.end is null ? new FrameBound(FrameBoundKind.CurrentRow) : BuildFrameBound(ctx.end); + if (start.Kind == FrameBoundKind.UnboundedFollowing) + throw new SqlParseException("A window frame cannot start at UNBOUNDED FOLLOWING."); + if (end.Kind == FrameBoundKind.UnboundedPreceding) + throw new SqlParseException("A window frame cannot end at UNBOUNDED PRECEDING."); + if (start.Kind > end.Kind) + throw new SqlParseException("A window frame cannot start after it ends."); + + FrameUnit unit = ctx.unit.Type switch + { + ROWS => FrameUnit.Rows, + RANGE => FrameUnit.Range, + _ => FrameUnit.Groups, + }; + FrameExclusion exclusion = ctx.exclusion switch + { + null => FrameExclusion.NoOthers, + { } e when e.CURRENT() is not null => FrameExclusion.CurrentRow, + { } e when e.GROUP() is not null => FrameExclusion.Group, + { } e when e.TIES() is not null => FrameExclusion.Ties, + _ => FrameExclusion.NoOthers, // NO OTHERS + }; + return new WindowFrame(unit, start, end, exclusion); + } + + private static FrameBound BuildFrameBound(FrameBoundContext ctx) + { + if (ctx.CURRENT() is not null) + return new FrameBound(FrameBoundKind.CurrentRow); + bool preceding = ctx.direction.Type == PRECEDING; + return ctx.UNBOUNDED() is not null + ? new FrameBound(preceding ? FrameBoundKind.UnboundedPreceding : FrameBoundKind.UnboundedFollowing) + : new FrameBound(preceding ? FrameBoundKind.Preceding : FrameBoundKind.Following, BuildExpression(ctx.offset)); + } /// A function name: an identifier, or the LEFT/RIGHT keyword tokens as Left()/Right(). private static string FunctionName(FunctionNameContext ctx) => @@ -931,8 +1116,6 @@ private static string FunctionName(FunctionNameContext ctx) => private static Expression BuildColumn(ColumnRefContext ctx) => new ColumnReference(OptionalIdentifier(ctx.qualifier), Identifier(ctx.name)); - /// Lowers x [NOT] BETWEEN lo AND hi to (x >= lo AND x <= hi) (negated for NOT), - /// so no dedicated node is needed and the evaluator handles it via the comparison operators. /// x IN (a, b, …) becomes a flat evaluated iteratively — NOT a /// deep (x = a) OR (x = b) OR … tree, which recurses once per item and overflows the stack when EF Core /// inlines a "huge number of values" Contains (thousands of constants). The evaluator reproduces the same @@ -944,24 +1127,74 @@ private static Expression BuildIn(InExprContext ctx) return new InListExpression(value, items, ctx.not is not null); } - private static Expression BuildBetween(BetweenExprContext ctx) + private static Expression BuildBetween(BetweenExprContext ctx) => + new BetweenExpression(BuildExpression(ctx.val), BuildExpression(ctx.lo), BuildExpression(ctx.hi), ctx.not is not null); + + /// Parses an Access #…# date literal (e.g. #1/1/1997#, month/day/year) to a + /// . A time without a date is on 1899-12-30, day zero (verified vs ACE: #13:45:30# is + /// 1899-12-30 13:45:30), not on today. + private static DateTime ParseDate(string text) { - Expression value = BuildExpression(ctx.val), lo = BuildExpression(ctx.lo), hi = BuildExpression(ctx.hi); - Expression range = new BinaryExpression(BinaryOperator.And, - new BinaryExpression(BinaryOperator.GreaterThanOrEqual, value, lo), - new BinaryExpression(BinaryOperator.LessThanOrEqual, value, hi)); - return ctx.not is null ? range : new UnaryExpression(UnaryOperator.Not, range); + DateTime value = DateTime.Parse(text.Trim('#'), CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault); + return value.Date == DateTime.MinValue ? new DateTime(1899, 12, 30).Add(value.TimeOfDay) : value; } - /// Parses an Access #…# date literal (e.g. #1/1/1997#, month/day/year) to a - /// . - private static DateTime ParseDate(string text) => - DateTime.Parse(text.Trim('#'), CultureInfo.InvariantCulture); + /// The exact value of a number written without an exponent; none for 1E5 or 1.5E2, or past a Decimal. + private static decimal? WrittenDecimal(string text) => + text.IndexOfAny(['E', 'e']) < 0 + && decimal.TryParse(text, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out decimal value) + ? value : null; + + /// + /// A minus written directly against a number that begins with a digit is part of that number, which is then a single + /// operand binding tighter than ^ (verified vs ACE: -2 ^ 2 is 4 and 2 ^ -2 ^ 2 is 0.0625, while + /// - 2 ^ 2, -(2) ^ 2 and -.5 ^ 2 are -4, -4 and -0.25). The parser gives the minus the powers + /// that follow, so this returns the signed number and the exponents to apply to it in turn, or null when the minus + /// is an ordinary negation. + /// + private static (LiteralExpression Number, List Exponents)? SignedNumber(NegateExprContext negation) + { + if (negation.op.Type != MINUS) + return null; + + var exponents = new List(); + ExpressionContext leftmost = negation.expression(); + while (leftmost is PowExprContext power) + { + exponents.Insert(0, power.right); + leftmost = power.left; + } + if (leftmost is not PrimaryExprContext primary || primary.primary() is not LiteralPrimaryContext literalPrimary) + return null; + + LiteralContext literal = literalPrimary.literal(); + string text = literal.GetText(); + if (literal is not (IntLiteralContext or NumberLiteralContext) + || literal.Start.StartIndex != negation.op.StopIndex + 1 + || !char.IsDigit(text[0])) + return null; + + return (BuildNumber("-" + text, literal is IntLiteralContext ? INTEGER_LITERAL : NUMBER_LITERAL), exponents); + } + + private static Expression PowerChain(Expression @base, IEnumerable exponents) => + exponents.Aggregate(@base, (result, exponent) => + new BinaryExpression(BinaryOperator.Power, result, BuildExpression(exponent))); + + private static Expression BuildPower(PowExprContext power) => + power.right is NegateExprContext negation && SignedNumber(negation) is { } signed + ? PowerChain(new BinaryExpression(BinaryOperator.Power, BuildExpression(power.left), signed.Number), signed.Exponents) + : new BinaryExpression(BinaryOperator.Power, BuildExpression(power.left), BuildExpression(power.right)); + + /// A number literal, with the sign when a minus is written against it. + private static LiteralExpression BuildNumber(string text, int tokenType) => tokenType == INTEGER_LITERAL + ? new LiteralExpression(ParseInteger(text)) + : new LiteralExpression(double.Parse(text, CultureInfo.InvariantCulture), WrittenDecimal(text)); private static Expression BuildLiteral(LiteralContext ctx) => ctx switch { - IntLiteralContext i => new LiteralExpression(ParseInteger(i.GetText())), - NumberLiteralContext n => new LiteralExpression(double.Parse(n.GetText(), CultureInfo.InvariantCulture)), + IntLiteralContext i => BuildNumber(i.GetText(), INTEGER_LITERAL), + NumberLiteralContext n => BuildNumber(n.GetText(), NUMBER_LITERAL), HexLiteralContext h => new LiteralExpression(ParseHexBytes(h.GetText())), StringLiteralContext s => new LiteralExpression(Unquote(s.GetText())), DateLiteralContext d => new LiteralExpression(ParseDate(d.GetText())), @@ -990,6 +1223,11 @@ private static BinaryExpression Binary(Antlr4.Runtime.IToken op, ExpressionConte MOD => BinaryOperator.Modulo, BACKSLASH => BinaryOperator.IntDivide, AMP => BinaryOperator.Concat, + AND => BinaryOperator.And, + OR => BinaryOperator.Or, + XOR => BinaryOperator.Xor, + EQV => BinaryOperator.Eqv, + IMP => BinaryOperator.Imp, BAND => BinaryOperator.BitAnd, BOR => BinaryOperator.BitOr, BXOR => BinaryOperator.BitXor, diff --git a/src/LibRed/README.md b/src/LibRed/README.md index 5f0c7e216..e90e2642a 100644 --- a/src/LibRed/README.md +++ b/src/LibRed/README.md @@ -14,7 +14,7 @@ LibRed reads and writes the file format directly. > the repo-root `CLAUDE.md`. Record only facts verified against real files or Access's own engine. > > **SQL surface:** [`docs/functions.md`](docs/functions.md) catalogs the supported VBA/Access functions -> (usable in `SELECT`/`WHERE`/`ORDER BY`/`DEFAULT`/`CHECK`); [`docs/format/page-02c-default-values.md`](docs/format/page-02c-default-values.md) +> and LibRed's extended ones (usable in `SELECT`/`WHERE`/`ORDER BY`/`DEFAULT`/`CHECK`); [`docs/format/page-02c-default-values.md`](docs/format/page-02c-default-values.md) > covers column `DEFAULT` semantics. ## Projects @@ -98,7 +98,17 @@ Treat the number as of its date — an EF Core version bump moves it. round-trips**; AutoNumber generation and high-water tracking (including the two's-complement wrap past `int32`, which ACE does not treat as an error either); unique-index statistics; allocation through the global free-pages map; `MSysObjects` / `MSysACEs` catalog rows. `UPDATE`/`DELETE` write in place, relocate - rows that no longer fit, maintain every index, and reclaim LVAL pages. + rows that no longer fit, maintain every index, and reclaim LVAL pages. Freed pages go back to that map + when the database closes, as ACE holds them — only an `UPDATE`'s replaced long value is reusable at once. + `DROP TABLE` frees every page the table owns: data and long-value pages (each Memo/OLE column's hang off a + per-column usage map, and are most of a memo-heavy table), every page of every index, the TDEF and its + continuation pages, and a reference-form map's bitmap pages. It retires each map's records from their + holder page in ACE's order and frees the holder once no other map's row is left, marks the released TDEF + `0x08`, and at close sizes the released-pages map — growing it, moving its window or converting it to + reference form — as Access does. Whole-file diffs against ACE drops of the same tables — memo, indexed, + multi-page-index, 255-column and reference-map tables, and across sessions — are **byte-identical** apart + from page 0's commit slot and the catalog's own pages: MSysObjects' wall-clock dates, and index pages whose + entries match but which LibRed compacts harder after removing them (spec §10.4a). - **Encryption** — read *and* write, in every scheme the format has: `DatabaseEncryption` sets, changes and removes passwords for Agile, Office Standard AES-256 and RC4 (selectable key length and hash), and the legacy Jet 4 database password **byte-identically to Access**; `SetJetEncoding` writes legacy RC4 page @@ -174,14 +184,33 @@ Treat the number as of its date — an EF Core version bump moves it. what extended mode generates against (see the EF Core section below): - `CROSS APPLY` / `OUTER APPLY` — a lateral join, with the right side re-evaluated per left row. ACE has no syntax for either. - - **Window functions** — `ROW_NUMBER()`, `RANK()` and `DENSE_RANK()` with - `OVER (PARTITION BY … ORDER BY …)`. `OVER` hangs off any function call, so adding another is a registry - entry rather than a grammar change. + - **Window functions** — `ROW_NUMBER()`, `RANK()`, `DENSE_RANK()`, `NTILE(n)`, `PERCENT_RANK()`, + `CUME_DIST()`, `LAG`/`LEAD(x [, offset [, default]])`, and `FIRST_VALUE`, `LAST_VALUE`, + `NTH_VALUE(x, n) [FROM FIRST | FROM LAST]` and the aggregates (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, Access's + `First`/`Last`, the statistical ones under their Access and standard names, the percentiles, and + `CORR`/`COVAR_*`/`REGR_*`, `LISTAGG`) with `OVER (PARTITION BY … ORDER BY … [frame])`; the one-argument + aggregates also take `DISTINCT`, and every aggregate `FILTER (WHERE …)`. Over a grouped query the windows + run over the groups `HAVING` keeps, so `RANK() OVER (ORDER BY SUM(x) DESC)` and `SUM(SUM(x)) OVER ()` rank + and total the groups. `LAG`, `LEAD` and the three value functions take + `RESPECT NULLS` or `IGNORE NULLS` before the `OVER`. The frame is the standard's: `ROWS`, `RANGE` or + `GROUPS`, `BETWEEN` any of `UNBOUNDED PRECEDING`, `n PRECEDING`, `CURRENT ROW`, `n FOLLOWING` and + `UNBOUNDED FOLLOWING`, with an optional `EXCLUDE CURRENT ROW | GROUP | TIES | NO OTHERS`. A `RANGE` offset + measures a number or date `ORDER BY` key (a date in days). Without a frame it is the default: with an ORDER BY + a running value to the current row and its peers, without one the whole partition. The words these clauses + add are not reserved, so a column named `Range` or `Current`, or a table named `Last`, still works + unbracketed. `OVER` hangs off any function call, so adding another function is a registry entry rather than + a grammar change. + - **Ordered-set aggregates** — `PERCENTILE_CONT(p)` and `PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY x [DESC])`, + grouped or over a window. `PERCENTILE_CONT` interpolates between numbers or dates; `PERCENTILE_DISC` returns + one of the values, so it also takes text. `LISTAGG([DISTINCT] x [, 'separator']) WITHIN GROUP (ORDER BY …)` + joins the values as text. + - **`FILTER (WHERE …)`** on any aggregate, grouped or windowed: `COUNT(*) FILTER (WHERE Amount > 100)`. - `FULL [OUTER] JOIN` — ACE offers only inner/left/right, and its query designer cannot express a full one. - **`OFFSET … ROWS FETCH NEXT … ROWS ONLY`** paging, where the count may be any expression, not just a literal. Access has only `TOP n`, and only with a literal. - - **Standard scalar syntax** ACE lacks: `CASE`, `COALESCE`, `NULLIF`, and the `VALUES` table value - constructor standing in for a query. + - **Standard scalar syntax** ACE lacks: `CASE`, `COALESCE`, `NULLIF`, `GREATEST`/`LEAST` (NULL arguments + ignored, as SQL Server and PostgreSQL treat them — extended mode translates `Math.Max`/`Math.Min` to + them), and the `VALUES` table value constructor standing in for a query. - **Set operations in a subquery predicate** — `IN (… UNION …)`, `EXISTS (… EXCEPT …)`, and a scalar subquery over a set operation. - **`ORDER BY` bound to the query expression**, so it applies to a whole set operation rather than to its @@ -215,9 +244,6 @@ Format-level detail on each on-disk gap lives in `docs/format/`. same blob, read by the same code, and *is* enforced. - **`AllowZeroLength` not modelled**, and column-level `CHECK` persistence is unprobed (its ACE storage differs from the table-level form). -- **`DROP TABLE` leaks until Compact** — multi-page TDEFs, non-root index pages, LVAL pages, and dedicated - usage-map pages aren't freed; byte-faithful **child-in-relationship** `DROP TABLE` (ACE cascades the FK; - LibRed requires dropping the FK first). - **Jet 3** format; strict **DAO Compact & Repair** compatibility (checklist captured — only relevant if targeting DAO C&R rather than "ACE opens + queries"). - **`CREATE TEMPORARY TABLE`** — parsed only to throw `NotSupportedException`. diff --git a/src/LibRed/docs/design/index-key-checksum.md b/src/LibRed/docs/design/index-key-checksum.md index eb5edf75c..1ae24a094 100644 --- a/src/LibRed/docs/design/index-key-checksum.md +++ b/src/LibRed/docs/design/index-key-checksum.md @@ -1,6 +1,6 @@ # The index-key truncation checksum -*Working note, not spec. The verified layout lives in `docs/format/page-03-04-index-btree.md` §10.4; this is +*Working note, not spec. The verified layout lives in `docs/format/page-03-04-index-btree.md` §10.4b; this is how it was arrived at, kept because the reasoning is worth more than the one line it produced.* **Status: settled.** Every question this note was opened with has an answer. Nothing here is outstanding. diff --git a/src/LibRed/docs/format/README.md b/src/LibRed/docs/format/README.md index 47e793505..ed5e5b547 100644 --- a/src/LibRed/docs/format/README.md +++ b/src/LibRed/docs/format/README.md @@ -2,12 +2,9 @@ This is LibRed's own specification of the Microsoft Jet 4 / ACE (Access `.mdb` / `.accdb`) on-disk format. **Every offset and structure here has been verified byte-for-byte against -real database files** (the Northwind ACE-2007 sample, a generated 200-column wide table, -and a generated ~150 MB large table) and cross-checked against mdbtools (its `HACKING.md`) -and Jackcess. Several structures are additionally verified from the **write** side: LibRed -produces them and Access's own OLE DB engine reads the result back (a LibRed-inserted row is -found by an Access indexed primary-key seek; encoded index keys match Access's stored bytes). -Where something is assumed or unverified, it says so explicitly. +real database files** and cross-checked against mdbtools (its `HACKING.md`). Several structures +are additionally verified from the **write** side: LibRed produces them and Access's own OLE DB +engine reads the result back. Where something is assumed or unverified, it says so explicitly. Unless noted, everything here describes **Jet 4 and ACE (12/14/16/17)**, which share one structural layout. **Jet 3** (Access 97) differs in many of these and is *not yet @@ -39,6 +36,8 @@ Implemented by `src/LibRed/LibRed.Core/`. The canonical offsets live in | [page-02e-calculated-columns.md](page-02e-calculated-columns.md) | **Calculated column** *semantics* — the expression language ACE accepts, when the cached result is recomputed, and what DDL may do to one (engine behaviour; the descriptor and value envelope are in [page-02b](page-02b-columns.md)) | | [page-03-04-index-btree.md](page-03-04-index-btree.md) | Index B-tree pages (types `0x03` node / `0x04` leaf): header, entries, prefix compression, key encoding, splitting | | [page-05-usage-maps.md](page-05-usage-maps.md) | Per-table owned/free usage maps, `0x05` bitmap pages, and the global free-pages map (allocation) | +| [page-08-released-tdef.md](page-08-released-tdef.md) | Released table-definition page (type `0x08`): what `DROP TABLE` leaves behind | +| [page-09-released-long-value.md](page-09-released-long-value.md) | Released long-value page (type `0x09`): a packed LVAL page emptied of its values | | [long-values.md](long-values.md) | Memo / OLE long values, LVAL pages, and the per-column usage-map list | | [data-types.md](data-types.md) | Data-type codes and their decode, plus compressed Unicode | | [system-catalog.md](system-catalog.md) | `MSysObjects` / `MSysACEs` / `MSysQueries` / `MSysRelationships`, the `LvProp` property blob, views & procedures, relationships | @@ -65,13 +64,19 @@ catalogued one level up in [`../functions.md`](../functions.md). | `0x03` | Index B-tree node (intermediate) | `IndexCursor` | [page-03-04](page-03-04-index-btree.md) | | `0x04` | Index B-tree leaf | `IndexCursor` | [page-03-04](page-03-04-index-btree.md) | | `0x05` | Page-usage bitmap | `UsageMap` | [page-05](page-05-usage-maps.md) | + | `0x08` | Released table definition (a dropped table's TDEF) | `PageType.ReleasedTableDefinition` | [page-08](page-08-released-tdef.md) | + | `0x09` | Released long-value page (emptied of its packed values) | `PageType.ReleasedLongValuePage` | [page-09](page-09-released-long-value.md) | + + `0x08` and `0x09` mark pages that have been **given back**. Neither needs handling on read — allocation + selects on the global free map, not on this byte — but both are written, so a file LibRed produces carries + the same markers Access would. --- ## Section map Cross-references throughout use the original **§-numbers** from the single-file spec. This -table says which file each section now lives in. +table says which file each section lives in. | § | Section | File | | --- | --- | --- | @@ -93,7 +98,8 @@ table says which file each section now lives in. | §7 | Compressed Unicode | [data-types.md](data-types.md) | | §8 | Long values (Memo / OLE) | [long-values.md](long-values.md) | | §9 | Usage maps | [page-05-usage-maps.md](page-05-usage-maps.md) | -| §9.1 | Global free-pages map | [page-05-usage-maps.md](page-05-usage-maps.md) | +| §9.1 | Global usage maps (free and released pages) | [page-05-usage-maps.md](page-05-usage-maps.md) | +| — | Released pages (`0x08`, `0x09`) | [page-08](page-08-released-tdef.md) / [page-09](page-09-released-long-value.md) — no §-number | | §10 | Index B-tree pages | [page-03-04-index-btree.md](page-03-04-index-btree.md) | | §11 | System catalog | [system-catalog.md](system-catalog.md) | | §12 | Version differences | this README (below) | @@ -121,12 +127,7 @@ with Jet 4/ACE defaults; a future `Jet3Format` overrides the ones that differ. ## Provenance -Verified against: `Northwind.accdb` (ACE 2007), a generated 200-column ACCDB (multi-page TDEF), -and a generated ~150 MB ACCDB (reference usage map). Cross-referenced with mdbtools -(`HACKING.md`, and its `table.c` / `data.c` / `index.c`) and Jackcess (`TableImpl`, `ColumnImpl`, -`IndexData`, `IndexCodes`) — consulted upstream, not vendored here. The LibRed test suite -(`test/LibRed.Core.Tests/`) pins these structures, including whole-database golden dumps. -Write-side structures (row insertion, order-preserving key encoding, leaf-entry layout) are -additionally cross-checked against Access's own engine via OLE DB: insert the same row through -LibRed and through Access, then confirm the row sets match and that Access seeks the -LibRed-written index entry. +Cross-referenced with mdbtools (`HACKING.md`, and its `table.c` / `data.c` / `index.c`) — consulted +upstream, not vendored here. Write-side structures (row insertion, order-preserving key encoding, leaf-entry +layout) are additionally verified against Access's own engine via OLE DB: Access reads LibRed-written rows +and seeks LibRed-written index entries. diff --git a/src/LibRed/docs/format/appendix-structures.md b/src/LibRed/docs/format/appendix-structures.md index 83d7d51a6..083af9730 100644 --- a/src/LibRed/docs/format/appendix-structures.md +++ b/src/LibRed/docs/format/appendix-structures.md @@ -1,7 +1,7 @@ # Appendix — on-disk structures (quick reference) Field-layout tables for every on-disk structure, with **no prose** — a fast lookup. Each -structure links to the file with the verified detail (edge cases, write rules, provenance). +structure links to the file with the verified detail (edge cases, write rules). All integers little-endian unless noted; offsets are hex, relative to the structure's start. --- @@ -16,6 +16,8 @@ All integers little-endian unless noted; offsets are hex, relative to the struct | `0x03` | Index B-tree node | | `0x04` | Index B-tree leaf | | `0x05` | Page-usage bitmap | +| `0x08` | Released table definition — a dropped table's TDEF, otherwise unchanged | +| `0x09` | Released long-value page — a packed LVAL page emptied of its values | --- @@ -27,17 +29,18 @@ All integers little-endian unless noted; offsets are hex, relative to the struct | `0x01` | 3 | Unknown (observed `01 00 00`, constant) | | `0x04` | 15 | Format id ASCII: `Standard Jet DB` (`0x00`/`0x01`) / `Jet System DB` (`0x01`) / `Standard ACE DB` (`0x02`+) | | `0x13` | 1 | NUL terminator of the id string | -| `0x14` | 1 | Version byte (`0x00` Jet3, `0x01` Jet4, `0x02` ACE12, `0x03` ACE14, `0x04` ACE15/2013 reserved-unemitted, `0x05` ACE16, `0x06` ACE17) | -| `0x15` | 1 | Version minor/update byte (`0x01` on ACE14/Access 2010, else `0x00`) | +| `0x14` | 1 | Version byte (`0x00` Jet3, `0x01` Jet4, `0x02` ACE12, `0x03` ACE14, `0x04` ACE15/2013 never emitted and refused by ACE, `0x05` ACE16, `0x06` ACE17) | +| `0x15` | 1 | Version minor byte (`0x01` on a file created as ACE14/Access 2010, else `0x00`; a version raise writes `0x00`) | | `0x16` | 2 | Unknown (zero) | | `0x18`–`0x98` | 128 | **Obfuscated header** — XOR'd with the fixed 128-byte mask; the `0x18`–`0x72` fields below are offsets into it (Jet3 masks 126) | -| `0x18`, `0x1C` | 4+4 | Fixed constants `0x00000100` / `0x00000101` | +| `0x18` | 4 | Global free-pages map pointer `[row:1][page:3]` (`0x00000100` = page 1 row 0) | +| `0x1C` | 4 | Global released-pages map pointer `[row:1][page:3]` (`0x00000101` = page 1 row 1) | | `0x20`–`0x2C` | 4×4 | Catalog bootstrap pointers — `MSysObjects`/`MSysACEs`/`MSysQueries`/`MSysRelationships` TDEF pages (`2`/`3`/`4`/`5`); `0x20` = catalog root | | `0x30`–`0x3B` | 12 | Reserved (zero) | | `0x3C` | 2 | ANSI code page (LE; `0x04E4` = 1252) | | `0x3E` | 4 | Database/encryption key (`0` = not encrypted) | | `0x42` | 40 | Password (Jet4; Jet3 = 20) — also XOR `(int)creationDate` | -| `0x6A` | 4 | Fixed constant `0x000011A6` | +| `0x6A` | 4 | Creating engine's build number (`0x000011A6` = 4518 on everything ACE writes; Jet 4 files carry their `msjet40.dll` build) | | `0x6E` | 4 | Collation, a 32-bit LCID with the version in its top byte: LANGID (2, LE), sort id at `0x70`, sort-order version at `0x71` (`0` legacy table, `1` Access-2010) | | `0x72` | 8 | Creation timestamp — OLE `double` (days from 1899-12-30) | | `0x98` | 4 | Fixed constant `0x00000654` (past the masked window) | @@ -58,7 +61,7 @@ All integers little-endian unless noted; offsets are hex, relative to the struct | `0x01` | 1 | Flags (`0x01`) | | `0x02` | 2 | Free space | | `0x04` | 4 | Owning TDEF page — or ASCII `LVAL` (`0x4C41564C`) for long-value pages | -| `0x08` | 4 | Jet4-only, zero observed | +| `0x08` | 4 | Jet4-only; zero except on the **first page of a long-value chain**, where it is the chain stamp matching the pointing descriptor's `0x08` | | `0x0C` | 2 | Row count | | `0x0E` | 2×N | Row-slot directory | @@ -68,7 +71,7 @@ All integers little-endian unless noted; offsets are hex, relative to the struct ``` [colCount:2 = maxColumnId+1] [fixed data] [var data] [varOffsetTable:(numVar+1)×2] [numVar:2] [nullBitmap:ceil(colCount/8)] ``` -Variable section (`varOffsetTable`+`numVar`) omitted when the table has no variable columns. Null bitmap keyed by column id (set = present); dead ids' bits set. Booleans carry no data (the bit *is* the value). +Variable section (`varOffsetTable`+`numVar`) omitted when the table has no variable columns. Null bitmap keyed by column id (set = present); dead ids' bits set. Booleans carry no data (the bit *is* the value). With the variable section omitted, `fixed data` is padded to a **minimum of 2 bytes** (a floor, not an alignment — an odd 3-byte region stays 3), making 5 the shortest record; ACE misreads anything shorter. --- @@ -113,8 +116,8 @@ Variable section (`varOffsetTable`+`numVar`) omitted when the table has no varia | `0x05` | 2 | Column id | | `0x07` | 2 | Variable-table index — on a **fixed** column the running count of preceding variable columns, **NOT `0`**; on a variable column its own slot index, which follows the `0x2B` high-water ([page-02b §3.4](page-02b-columns.md)) | | `0x09` | 2 | Column number — a second copy of the id `0x05` on a **user** table, but **`0`** on the tables the engine writes for itself; unchanged by an `ALTER COLUMN` that burns a new id at `0x05` ([page-02b §3.4](page-02b-columns.md)) | -| `0x0B` | 1 | Precision (Decimal) — else locale low byte `0x09` | -| `0x0C` | 1 | Scale (Decimal) — else locale high byte `0x04` | +| `0x0B` | 1 | Precision (Decimal) — else collation LANGID low byte (`0x09` en-US) | +| `0x0C` | 1 | Scale (Decimal) — else collation LANGID high byte (`0x04` en-US) | | `0x0D` | 1 | Collation sort id — the LCID's high word (`0x01` = an alternate sort order, e.g. Hungarian Technical) | | `0x0E` | 1 | Sort-order version (`0` legacy table, `1` Access-2010) | | `0x0F` | 1 | Flags: `0x01` fixed, `0x02` updatable, `0x04` auto-number, `0x40` auto-number GUID, `0x80` hyperlink | @@ -131,10 +134,11 @@ Nullability is **not** in the descriptor — it's the `Required` property in `Lv | Offset | Size | Meaning | | --- | --- | --- | -| `0x00` | 16 | Reserved (zero) | +| `0x00` | 4 | VBA error number, little-endian — `0` for a value or Null; an error envelope is 38 bytes | +| `0x04` | 12 | Reserved (zero in every row observed) | | `0x10` | 4 | Payload length, little-endian | | `0x14` | *n* | Payload — the value in its ordinary encoding | -| `0x14`+*n* | 3 | Padding (zero) | +| `0x14`+*n* | 3 | Padding (zero in every row observed) | The descriptor's type at `0x00` is a **promoted storage type**; the payload length says the real one (Int16+1 = Boolean, Int32+1 = Byte, Int32+2 = Int16, Double+4 = Single). Length `0` means Null, and the @@ -189,7 +193,7 @@ descriptor. | `0x00` | 4 | Little-endian word: length in bits 0–29; flags in bits 30–31 (`0x80000000` inline, `0x40000000` single LVAL page, `0x00000000` chain) | | `0x04` | 1 | Row | | `0x05` | 3 | Page | -| `0x08` | 4 | Reserved | +| `0x08` | 4 | **Chain stamp**, chained form only (zero on inline and single-page): must equal the first chain page's header `0x08`, or ACE refuses the record. ACE stamps `GetTickCount()`; the value is arbitrary, the agreement is not. LibRed stamps both and verifies them on read | **Per-long-value-column usage-map list entry (10 bytes; list ends at `col_num == 0xFFFF`):** @@ -206,7 +210,8 @@ descriptor. **Inline (type `0x00`):** `[0x00][startPage:4][bitmap…]` — bit `i` ⇒ page `startPage+i` owned. **Reference (type `0x01`, 69 bytes):** `[0x01][17 × 4-byte bitmap-page pointers]`. **Bitmap page (type `0x05`):** header `[0x05][0x01][0][0]`, bitmap from offset 4. -Global free-pages map: **page 1, row 0**, inline or reference — set bit = **free** (opposite of a table map). +Global maps, located by page 0: free pages at `0x18` (page 1 row 0 as ACE writes it) — set bit = **free** +(opposite of a table map); released pages at `0x1C` (page 1 row 1) — set bit = freed, not reusable until close. --- @@ -261,7 +266,7 @@ Global free-pages map: **page 1, row 0**, inline or reference — set bit = **fr ## Limits -Three distinct kinds (the useful mental model): **structural** — the byte layout can't represent more, so +Three distinct kinds: **structural** — the byte layout can't represent more, so guard in the serializer; **engine constant** — a fixed-size buffer in ACE's reader (the format holds more), so guard with a validator; **query-engine** — ACE's SQL-engine limits that LibRed deliberately exceeds. @@ -285,12 +290,12 @@ exists to beat and are deliberately **not** guarded. The limits above are all stated where they bind. These are not: a field in **one** structure fixes a ceiling that a writer of a **different** structure has to respect, and nothing in the second structure's layout says -so. Both bugs found in this class were silent — the write succeeds, the read succeeds, and the wrong row -comes back — so the table records how each ceiling is actually held, not merely that it exists. +so. Crossing one is silent — the write succeeds, the read succeeds, and the wrong row comes back — so the +table records how each ceiling is actually held, not merely that it exists. | Narrow field | Ceiling it imposes | How it is held | | --- | --- | --- | -| Index leaf entry addresses a row as `page << 8 \| row` — 1 byte of slot | **256 rows per data page**, though the row count at `0x0C` is 2 bytes and a 4 KB page fits far more | **Enforced.** `FindPageWithRoom` refuses a page at `RowPointer.MaxRowsPerPage`, covering both the insert path and `WriteHiddenRow` (a relocation target is named by the same pointer). Was a live bug: narrow all-fixed rows reached 314 per page, and every slot past 255 aliased another row | +| Index leaf entry addresses a row as `page << 8 \| row` — 1 byte of slot | **255 rows per data page**: the pointer allows 256 but **ACE writes at most 255**, not for space (a filled page keeps ~2,297 of 4,096 bytes free) and it drops the page from the free-pages map on reaching it | **Enforced at ACE's 255.** `FindPageWithRoom` refuses a page at `RowPointer.MaxRowsPerPage`, covering both the insert path and `WriteHiddenRow` (a relocation target is named by the same pointer). Overfilling costs more than indexed reads — ACE parses the full 16-bit count but caps at 256 slots, so it silently cannot see the rest of the page's rows | | Long-value descriptor names its row in 1 byte (`d[4]`) | **256 rows per LVAL page** | **Enforced** in `TryAppend`. Unreachable in practice — a payload ≤ 64 bytes inlines, and the free-map drop at `MinLvalRow` caps a page near 108 rows even for the smallest thing that can arrive (a 33-character memo compressed to 35 bytes; compression is applied *after* the inline test, so the floor is below the 65 bytes the inline limit suggests) | | Page numbers are 3 bytes in the TDEF usage-map pointer, the long-value descriptor (`d[5..7]`) and an LVAL chunk's next-pointer | **page < 2²⁴** (16,777,216) | **Safe with 32× headroom**, because `PageChannel.WritePage` enforces the 2 GiB file limit at 524,288 pages. The 24-bit fields are never the binding constraint | | Usage-map pointer names its record row in 1 byte | **256 records per usage-map page** | **Safe by a louder guard.** A record is 69 bytes, so `AppendEmptyUsageMapRow`'s space check admits 57 and refuses the 58th — 4.5× tighter than the byte — and it throws rather than truncating | @@ -298,5 +303,5 @@ comes back — so the table records how each ceiling is actually held, not merel | Index page entry mask spans `0x1B`–`0x1E0` (453 bytes = 3,624 bits) | one bit per byte of entry data, which starts at `0x1E0` | **Exact fit, not slack**: entry data tops out at `4096 − 0x1E0` = 3,616 bytes, so the highest bit lands in the mask's last byte. `EntryDataOffset` is evidently chosen for this | | Row slot offset is 13 bits (`0x1FFF` = 8,191) | offsets within a 4 KB page | Safe by 2×; the mask exists for the two flag bits above it | -The pattern worth carrying: **a ceiling is only safe if something refuses to cross it, or if a tighter guard -fires first and says so.** "The arithmetic doesn't reach it" is the state both live bugs were in. +The rule: **a ceiling is only safe if something refuses to cross it, or if a tighter guard fires first and +says so.** "The arithmetic doesn't reach it" is not a guard. diff --git a/src/LibRed/docs/format/data-types.md b/src/LibRed/docs/format/data-types.md index 4292a29e6..6caac5b52 100644 --- a/src/LibRed/docs/format/data-types.md +++ b/src/LibRed/docs/format/data-types.md @@ -28,9 +28,7 @@ > **Unmodelled codes are read, not refused.** `0x0D`, `0x0E` and `0x11` are held open in `JetDataType` as > `Unknown*` placeholders: a TDEF carrying one still parses, the column decodes to its **raw bytes**, and only > *writing* such a column is refused. This is not tidiness — the catalog reads **every** table's definition, -> so one unrecognised column previously made the whole database unopenable. Measured: three of four real Jet 4 -> `.mdb` files taken off the web failed to open at all for this reason, and read cleanly afterwards -> (69 tables, ~204k rows across the four). +> so a reader that refuses one unrecognised column cannot open the database at all. > > **`0x11` is the only one seen in the wild**, and never on a user column: it is always > `MSysAccessObjects.Data`, fixed length, 3992 bytes. The contents are chunks of an **OLE Compound File** @@ -40,12 +38,10 @@ > > **It belongs to the legacy object-storage table, not to any feature.** Access later replaced > `MSysAccessObjects` with `MSysAccessStorage`, which uses modelled types, and a file has one or the other. -> The discriminator is **not** the format version — every Jet 4 `.mdb` measured carries page-0 version byte -> `0x01`, and they split either way. It is the generation Access chose **when it created the database**, -> recorded as the `AccessVersion` property on the `MSysDb` object: `08.50` (Access 2000) uses the legacy -> store, `09.50` (Access 2002+) does not. Measured on two new databases from one Access install's New dialog, -> one in each format. Adding a type-library reference to a current `.accdb` does **not** produce it, tested -> because the readable chunk made that the obvious suspicion. +> The discriminator is **not** the format version — Jet 4 `.mdb` files with page-0 version byte `0x01` split +> either way. It is the generation Access chose **when it created the database**, recorded as the +> `AccessVersion` property on the `MSysDb` object: `08.50` (Access 2000) uses the legacy store, `09.50` +> (Access 2002+) does not. Adding a type-library reference to a current `.accdb` does **not** produce it. > > So a current Access still writes `0x11` today if asked for a new Access 2000 database — but only that way. > A file created by anything else (DAO, LibRed) gets `MSysAccessStorage` when Access first opens it, whatever @@ -58,57 +54,74 @@ GUID, date, or decimal codec. Text, Binary, Memo/OLE descriptors, and Complex va variable-length. A width mismatch is treated as row corruption (`InvalidDataException`) rather than being allowed to fail incidentally inside a primitive decoder. +**A `NUMERIC`/`DECIMAL` column's declared precision is a contract the payload cannot express, and ACE enforces +it on write.** The stored form is the same 17 bytes — a sign byte plus a 128-bit magnitude — whatever the column +declares, so a 20-digit value occupies a `DECIMAL(18,4)` exactly as comfortably as a 2-digit one and nothing on +disk records that it is out of contract. ACE therefore checks the value against the declaration instead, and +refuses what will not fit with *"The decimal field's precision is too small to accept the numeric you attempted +to add."* The permitted magnitude follows the standard rule: `p` total digits with `s` after the point, so at +most `p − s` before it, and `DECIMAL(18,4)` accepts up to `99999999999999.9999` and refuses `10^14`. + +The enforcement covers **every path that can put a value in the column** — `INSERT`, `UPDATE`, +`INSERT … SELECT`, and an `ALTER COLUMN` that *narrows* the declaration over rows already stored. That last one +is what makes it an invariant over the whole column rather than a filter on one statement: ACE will not shrink +a declaration to something its existing data would violate. + +> Scale is treated differently and is **not** refused: excess decimals are coerced, and **ACE truncates toward +> zero**. `1.23456` into a `DECIMAL(18,4)` stores `1.2345`, `1.99999` stores `1.9999`, `-1.23455` stores +> `-1.2345` — truncation in every case, over both midpoint parities and both signs. +> +> LibRed matches it in `JetTypeCodec.EncodeNumeric`, and `IndexKeyEncoder.EncodeFixedPoint` **must** quantise +> identically: the key is the same unscaled integer the row stores, so quantising one differently files a value +> under a number its row does not contain. Rounding (`decimal.Round(…, 0)`, `ToEven`) is the obvious +> implementation and is wrong: it stores `1.2346` and `2.0000` for ACE's `1.2345` and `1.9999`, and a row and +> index that round consistently with each other give no internal sign of it. + **`BIGINT` is variable-length despite being a fixed 8 bytes.** ACE puts it behind the row's variable offset table rather than in the fixed region — a descriptor carrying length 8 with the fixed flag clear (verified: a -column ACE created reads back `length=8 fixed=False`, and the row lays the value out at a variable-column -start offset). Its *index* key is unaffected — key encoding dispatches on the column's type, not on where +column ACE creates has `length=8 fixed=False`, and its row holds the value at a variable-column start +offset). Its *index* key is unaffected — key encoding dispatches on the column's type, not on where the row keeps the bytes; the encoding itself is [§10.4](page-03-04-index-btree.md). > **The reason to match is faithfulness, not readability — ACE honours the descriptor's fixed flag.** It has > to: its own `MSysComplexType_GUID` declares `Value` *fixed* while every GUID column its DDL creates is -> *variable*, so one engine reads both layouts routinely. Measured directly rather than argued: a **fixed** -> BIGINT, Currency and DateTime, and a **variable** Int32, Double, Currency and DateTime, all read back -> correctly through ACE — each with a variable column before the target and a fixed one after, so a value -> landing in the wrong region would have shifted its neighbours. (An earlier revision of this file claimed a -> fixed BIGINT would put the value "somewhere ACE does not look for it"; that was inferred from what ACE -> writes, never measured, and it is wrong.) `FixedFlagHonouredAccessTests`. +> *variable*, so one engine reads both layouts routinely. Verified: a **fixed** BIGINT, Currency and +> DateTime, and a **variable** Int32, Double, Currency and DateTime, all read back correctly through ACE. A +> fixed BIGINT is therefore *not* misread — the layout ACE writes is not the only one it looks for. **`GUID` is variable-length too — but only where ACE's DDL made it.** Every GUID column ACE's SQL creates -carries length 16 with the fixed flag clear, at one column or at 252 (it is not a fallback for wide tables), -and `SELECT … INTO` produces the same. ACE's **own system tables are the exception**: `MSysComplexType_GUID` -`Value` is *fixed*, verified across eight ACE-created fixtures, and `DatabaseCreator` reproduces that — so -"GUID is variable" is a rule about declarations, not about GUID storage everywhere. +carries length 16 with the fixed flag clear, whatever the table's width (it is not a fallback for wide +tables), and `SELECT … INTO` produces the same. ACE's **own system tables are the exception**: +`MSysComplexType_GUID` `Value` is *fixed* (verified), and `DatabaseCreator` reproduces that — so "GUID is +variable" is a rule about declarations, not about GUID storage everywhere. Unlike BIGINT this is not a wrong-value hazard: ACE reads a value back correctly from either layout -(verified with a variable column before the GUID and a fixed one after, so a misplaced value would have -shifted its neighbours). What it costs is record budget — 16 *fixed* bytes per column that ACE does not -spend, enough that a 250-column GUID table ACE creates without complaint exceeded the declared-record limit -in §3.4. `AccessTypeMapper` and `StatementExecutor.ColumnSpecFor` both declare it variable; -`GuidColumnStorageAccessTests` (in the Core and Engine suites) holds the measurements. +(verified). What it costs is record budget — 16 *fixed* bytes per column that ACE does not spend, enough to +push a 250-column GUID table that ACE creates without complaint past the declared-record limit in §3.4. +`AccessTypeMapper` and `StatementExecutor.ColumnSpecFor` both declare it variable. > **Writing one through ACE's OLE DB provider: not `DBTYPE_I8`.** An `OleDbType.BigInt` (20) parameter carries > **no** value into a Large Number column — every value fails with "data value could not be converted", zero > included — so the one type named for the job is the only one that cannot do it. Use **`Numeric` (131)**; > `Decimal` (14) and `Variant` (12) also round-trip the full range exactly. `VarNumeric` (139) is rejected > outright ("Type name is invalid"), and `Double` (5) is the trap: it succeeds quietly for small values and -> overflows near ±2⁶³. Measured against both extremes. EFCore.Jet's `JetLongTypeMapping.ConfigureParameter` -> already forces OLE DB 131 / ODBC 7 for its own `long` parameters, commented *"Using BigInt doesn't always -> work … When running in x64 it fails to convert"* — the same defect, found from the other direction and years -> earlier, though that mapping targets a `decimal(20,0)` column rather than a real `0x13` one. +> overflows near ±2⁶³. EFCore.Jet's `JetLongTypeMapping.ConfigureParameter` forces OLE DB 131 / ODBC 7 for its +> own `long` parameters for the same defect, though that mapping targets a `decimal(20,0)` column rather than a +> real `0x13` one. **New-type format versions — the two are NOT the same version**, which is the natural assumption and is wrong. **`BIGINT` (Large Number)** requires the **ACE 16 / Access 2016** format (`0x05`); **`DATETIME2` (Date/Time Extended)** requires the **ACE 17 / Access 2019+** format (`0x06`) — it arrived later (Access for -Microsoft 365). LibRed gates each accordingly (`AccessTypeMapper`). Both thresholds were measured from two -directions — see [page-00 §2](page-00-database.md), which owns the version byte and the experiment. `DATETIME2` is a fixed 42-byte ASCII string of -three colon-separated fields: the .NET **day number**, the count of **100-ns ticks within the +Microsoft 365). LibRed gates each accordingly (`AccessTypeMapper`). Both thresholds are verified — see +[page-00 §2](page-00-database.md), which owns the version byte. `DATETIME2` is a fixed 42-byte ASCII string +of three colon-separated fields: the .NET **day number**, the count of **100-ns ticks within the day**, and the fractional **precision** (e.g. `7`). The first two are zero-padded to 19 digits so that byte order equals chronological order (an order-preserving inline encoding). The value is `new DateTime(day * TicksPerDay + time)`; e.g. `…693593:…0:7` is the 1899-12-30 epoch and `…737590:…495300000000:7` is 2020-06-15 13:45:30. Sub-second precision (to 100 ns) is preserved. Those three fields occupy **41** characters (19 + 1 + 19 + 1 + 1); the 42nd byte is a **NUL (`0x00`), not a -space** — verified by reading rows ACE itself wrote (`… 3A 37 00`). The distinction is not cosmetic: the whole +space** — verified against rows ACE wrote (`… 3A 37 00`). The distinction is not cosmetic: the whole 42 bytes go into the index key verbatim, so a space there would put every key out of step with ACE's and make its seeks miss those rows. @@ -133,7 +146,7 @@ SQL DDL leaves it **clear**. > **The flag does not gate everything, and for a long value it gates the *page* case only.** A plain > `LONGTEXT` column — capable flag clear — still stores an **inline** value compressed. What the declaration -> buys is compression of a value that landed on a single LVAL page. Measured against ACE, ASCII throughout: +> buys is compression of a value that landed on a single LVAL page. Verified against ACE, ASCII throughout: > > | column | characters | form | on-disk length | > | --- | ---: | --- | ---: | @@ -146,21 +159,19 @@ SQL DDL leaves it **clear**. > > The 32/33 boundary also shows the form being chosen on the **uncompressed** length: 33 characters are 66 > bytes and go to a page, though they would compress to 35 and fit inline. An ordinary `Text` column does -> honour the flag, so this is specific to long values. LibRed required the flag everywhere and so stored -> every short ASCII memo at twice ACE's size — found by diffing row bytes, fixed in `JetTypeCodec`. -> `MemoCompressionAccessTests`. +> honour the flag, so this is specific to long values. A writer that requires the flag everywhere stores +> every short ASCII memo at twice ACE's size; LibRed follows ACE in `JetTypeCodec`. -**When a capable column actually compresses a value** (measured in `CompressedTextAccessTests` and -`LongTextStorageAccessTests`, and reproduced by LibRed byte-for-byte): +**When a capable column actually compresses a value** (verified against ACE, and reproduced by LibRed +byte-for-byte): - **A run of characters that fit one byte** (`<= 0xFF`, so Latin1, not just ASCII) is stored one byte per character. Runs are **split**, not all-or-nothing: a non-Latin1 character switches the value into 2-byte - mode rather than forfeiting compression for the whole of it, and both ACE and LibRed do this — see the - mixed-run note below, which supersedes the earlier reading. The exception is a 2-byte character with a - `0x00` low byte (`一` = `00 4E`), which is indistinguishable from the mode switch and does forfeit the - whole value. + mode rather than forfeiting compression for the whole of it — see the mixed-form note below. The exception + is a 2-byte character with a `0x00` low byte (`一` = `00 4E`), which is indistinguishable from the mode + switch and does forfeit the whole value. - **It must save space.** The marker costs 2 bytes, so 1- and 2-character values stay UTF-16 (2 + N < 2N only - from N = 3). Verified at each of 1, 2 and 3 characters. + from N = 3). - **A chained long value is never compressed.** Compression is decided *after* the storage form, and the form — inline, single page or chained — is chosen on the **uncompressed** UTF-16 length. So an inline Memo compresses (whatever the capable flag says), a single-page one compresses only on a `WITH COMPRESSION` @@ -172,10 +183,7 @@ SQL DDL leaves it **clear**. > mid-string: after the `FF FE` marker the value starts in 1-byte mode and every `0x00` byte at a character > boundary switches mode. `café中` is stored as `FF FE 63 61 66 E9 00 2D 4E` — "café", a switch, then 中. > -> This was previously recorded here as technically possible but unreproducible. The attempt that failed used -> 1,000 ASCII characters plus one CJK — far too long to stay inline, so it went to an LVAL page and was not -> compressed at all. The form appears on short **inline** values, and ACE emits it under two conditions, -> both measured: +> ACE emits it under two conditions: > > - **It must strictly save space.** `abc中` is 8 bytes either way, so it stays UTF-16; `café中` saves one > byte and is mixed. @@ -184,26 +192,21 @@ SQL DDL leaves it **clear**. > while `aaaaa中` is mixed. One such character forfeits the form for the whole value. > > It is **not** limited to inline values: 1,000 ASCII characters plus one `中` on an LVAL page store as -> 1,005 bytes (`FF FE` + 1,000 + switch + 2) where the same shape with `一` stores 2,002. So "one -> incompressible character forfeits the whole value" holds only for the *ambiguous* ones — the original -> reading of that experiment was an artefact of the character it happened to use. +> 1,005 bytes (`FF FE` + 1,000 + switch + 2) where the same shape with `一` stores 2,002 (plain UTF-16, the +> saving thrown away). So "one incompressible character forfeits the whole value" holds only for the +> *ambiguous* ones. The character's position is irrelevant in both cases; ACE 12.0 and 16.0 agree byte for +> byte. > -> Together those are what make toggling on `0x00` unambiguous for a reader. **LibRed used to decode the -> whole payload as a single Latin1 run**, silently returning `café\0-N` for a value Access wrote — wrong -> data, no error, and reachable by ordinary mixed-script text since Access's UI defaults Unicode Compression -> to Yes. `JetTypeCodec.DecodeText` now honours the switches, and `EncodeCompressed` emits them. +> The two conditions above are what make toggling on `0x00` unambiguous for a reader. **A reader that decodes +> the whole payload as a single Latin1 run** silently returns `café\0-N` for a value Access wrote — wrong data, no +> error, and reachable by ordinary mixed-script text since Access's UI defaults Unicode Compression to Yes. +> `JetTypeCodec.DecodeText` honours the switches, and `EncodeCompressed` emits them. > > **The two paths break an exact tie differently.** A long value takes the compressed form only when it is > strictly smaller; an ordinary `Text` column takes it when it is no larger. `ab中cd` is 10 bytes either way -> and comes back UTF-16 from a Memo but mixed from a `WITH COMPRESSION` Text column — measured on both. -> Under three characters nothing is compressed on either path, whatever the arithmetic says, which is what -> settles the all-Latin1 ties (`ab` is 4 bytes either way and stays UTF-16). -> -> **Only an *ambiguous* character forfeits the whole value** — one whose low byte is `0x00`, and so cannot be -> told from the mode switch (`一` = `00 4E`, `Ā` = `00 01`). `MixedCompressionAccessTests` measures both: -> 1,000 ASCII + one `一` stores as 2,002 bytes (plain UTF-16, the saving thrown away), while 1,000 ASCII + -> one `中` stores as 1,005 (compressed with a switch). Position is irrelevant in both cases. Checked by hand -> on ACE 12.0 and 16.0, which agree byte for byte; the test can only assert whichever is installed. +> and comes back UTF-16 from a Memo but mixed from a `WITH COMPRESSION` Text column. Under three characters +> nothing is compressed on either path, whatever the arithmetic says, which is what settles the all-Latin1 +> ties (`ab` is 4 bytes either way and stays UTF-16). --- @@ -225,18 +228,25 @@ Points verified against ACE that aren't obvious from that page: - **Bare `TEXT` → Memo** (long text); `TEXT(n)` → `varchar(n)` (a Jet quirk, ACE-verified). - Sized Text/Binary dimensions must be positive: Text is `1..255` characters and Binary is `1..510` bytes. - **`CHAR(n)` / `BINARY(n)` are FIXED-length columns; `TEXT(n)` / `VARBINARY(n)` are variable** — ACE's own DDL - produces both forms, so the fixed form is not a LibRed-only construct (`FixedWidthOverflowAccessTests`). + produces both forms, so the fixed form is not a LibRed-only construct. - **An over-long value is refused on both forms, with one message**: *"The field is too small to accept the - amount of data you attempted to add."* Measured on all four shapes above. The fixed form is the one worth - recording: because ACE stores fixed text space-padded to the full width, a writer that pads is one line away - from *truncating* the over-long case instead of refusing it, which is what LibRed did — silently storing - `'abc'` for `CHAR(3)` where the same value into `TEXT(3)` raised. The width check therefore belongs on the - encode path for fixed columns (`JetTypeCodec.EnsureFitsFixedWidth`, before padding) and on the shared - row-assembly path for variable ones (`RowEncoder.AssembleRow`, so the ALTER re-lay passes it too). + amount of data you attempted to add."* The fixed form is the one worth recording: because ACE stores fixed + text space-padded to the full width, a writer that pads is one line away from silently *truncating* the + over-long case instead of refusing it. The width check therefore belongs on the encode path for fixed + columns (`JetTypeCodec.EnsureFitsFixedWidth`, before padding) and on the shared row-assembly path for variable ones (`RowEncoder.AssembleRow`, so the ALTER re-lay passes it too). - **Narrowing an existing column is checked against its rows.** `ALTER TABLE … ALTER COLUMN c TEXT(5)` on a column holding wider values is refused rather than leaving rows that violate the declaration. -- `DECIMAL(p,s)` / `NUMERIC(p,s)` use precision `1..28` and scale `0..p`; LibRed rejects dimensions outside - those ACE/.NET decimal bounds before allocating or writing a table definition. +- `DECIMAL(p,s)` / `NUMERIC(p,s)` use precision `1..28` and scale `0..p`, and these are **ACE's own bounds, + refused at DDL with two distinct messages**: *"Invalid precision for decimal data type."* for `(0)`, `(0,0)` + and `(29)`, *"Invalid scale for decimal data type."* for `(5,7)`. `(1,0)` and `(28,28)` are both accepted, so + scale may equal precision — and `(28,28)` is usable rather than merely declarable: with no digits left in + front of the point it holds values below 1, keeping all 28 decimals, and both engines refuse `1`. + **Size-less `DECIMAL` and `NUMERIC` default to precision 18, scale 0** — the same kind of default as the + 255/510 above — while an explicit `(p)`/`(p,s)` is stamped exactly as written; the column is 17 bytes and + fixed-length either way. **A precision of 0 cannot be declared at all**, which fits ACE's own OLE DB reader + being unable to materialise such a column — a 0 leaves the value no declared shape to be read back into. + LibRed rejects out-of-range dimensions in `AccessTypeMapper` and, because a direct Core caller bypasses that, in `TdefBuilder` too; an unspecified + precision resolves to 18 on write rather than reaching the file as 0. - The grammar parses **two-word** type names (`CHARACTER VARYING`, `BIT VARYING`); three-word (`NATIONAL CHARACTER VARYING`) is not parsed yet. `HYPERLINK`/`XML`/`SQL_VARIANT`/`VARIANT`/`COMP` have no mapping (rejected, as ACE also rejects them). @@ -246,13 +256,13 @@ Points verified against ACE that aren't obvious from that page: ## Footnote — reading `DATETIME2` through ACE's own drivers -*Driver behaviour, not file format. Recorded here because it is the reason LibRed cannot cross-check this one -type against ACE the way it does every other type, and because it silently corrupts data in the wider repo.* +*Driver behaviour, not file format. Recorded here because ACE's own drivers cannot be used to cross-check +this one type the way every other type can, and because the OLE DB provider silently corrupts it.* **The bytes on disk are correct; ACE's OLE DB read path is broken in three independent ways.** Verified -2026-08-26 against **Access / Microsoft 365 version 2608 (build 20326.20100 Click-to-Run, Current Channel, -x64)** — i.e. the then-current shipping build, not an old redistributable. Values were inserted through ACE and -then read back three ways — ACE OLE DB, ACE ODBC, and LibRed reading the file directly: +against **Access / Microsoft 365 version 2608 (build 20326.20100 Click-to-Run, Current Channel, x64)** — a +shipping build, not an old redistributable. Values inserted through ACE, read back three ways — ACE OLE DB, +ACE ODBC, and LibRed reading the file directly: | literal | OLE DB (`Microsoft.ACE.OLEDB.16.0`) | ODBC (`ACEODBC.DLL`) | LibRed | | --- | --- | --- | --- | @@ -271,10 +281,9 @@ must be decoded by the caller (the same parse LibRed does). Note `OdbcConnection ### What the provider actually does -Measured with a consumer calling the OLE DB COM vtables directly — `CoCreateInstance` → `IDataInitialize` → -`IDBInitialize` → `IDBCreateSession` → `ICommandText` → `IColumnsInfo`/`IAccessor`/`IRowset`, every buffer -natively allocated. **No ADO, no `System.Data.OleDb`, no ODBC in the path**, so everything below is the -provider's own behaviour with nothing in between. +Observed through the OLE DB COM interfaces called directly (`IDataInitialize` → `IDBInitialize` → +`IDBCreateSession` → `ICommandText` → `IColumnsInfo`/`IAccessor`/`IRowset`, natively allocated buffers). +**No ADO, no `System.Data.OleDb`, no ODBC in the path**, so everything below is the provider's own behaviour. The column's `DBCOLUMNINFO`: @@ -303,8 +312,8 @@ with the converted struct, then reports `cbLength = 16, DBSTATUS_S_OK`. It never (6 bytes) and `DBTYPE_R8` (8 bytes) each get a 16-byte struct splatted at `obValue` regardless, then return `E_DATAOVERFLOW` — or, for `R8`, the flatly wrong `DBSTATUS_S_ISNULL`. -This is a genuine consumer buffer overrun, and it explains the `0xC0000374` / `0xC0000409` process crashes seen -under OLE DB reader churn: `System.Data.OleDb` places `obValue` at 16 in a 32-byte row buffer, so ACE writes 26 +This is a genuine consumer buffer overrun, and it explains `0xC0000374` / `0xC0000409` process crashes under +OLE DB reader churn: `System.Data.OleDb` places `obValue` at 16 in a 32-byte row buffer, so ACE writes 26 bytes off the end of a managed allocation. **2 — Its `DBTIMESTAMP` conversion is one month short.** The 16 bytes it wrote for `2021-03-04 05:06:07`: @@ -335,12 +344,7 @@ directly — what LibRed does — is not merely an alternative; it is the only c ### Why this has gone unnoticed Server-side comparison is unaffected — a `WHERE dt2 = #…#` matches correctly, because only *materialisation* -goes through the broken conversion. And it is live in the real provider stack, not just in a probe: -`AdHocMiscellaneousQueryJetTest` seeds **nine** `datetime2` columns (precisions 0–7), materialises all of them -in `Where_not_equals_DateTime_Now`, and is green — only because every seeded date is in **September**, which -merely shifts to August, and because the test asserts `Assert.Single` rather than any value. Changing one seeded -date to January makes it fail immediately with the `ArgumentOutOfRangeException` above (verified by doing it, -then reverting). So: predicates are right, corruption is silent outside January, Access itself never reads -through OLE DB, and the suites that do exercise the type check row counts rather than values. - -Test: `AceDateTime2UpgradeTests.LibRed_decodes_datetime2_values_that_ace_reads_back_wrongly`. +goes through the broken conversion. It is live in the ordinary `System.Data.OleDb` stack, not only in a raw +COM consumer: a materialised value from any month but January shifts back a month to a valid date, and only a +January value throws the `ArgumentOutOfRangeException` above. So: predicates are right, corruption is silent +outside January, and Access itself never reads through OLE DB. diff --git a/src/LibRed/docs/format/long-values.md b/src/LibRed/docs/format/long-values.md index 14671a6f2..e17e1cce2 100644 --- a/src/LibRed/docs/format/long-values.md +++ b/src/LibRed/docs/format/long-values.md @@ -11,24 +11,56 @@ The in-row value for a Memo/OLE column is a **12-byte descriptor**, not the data | `0x00` | 4 | Little-endian word: byte length in bits 0–29, flags in bits 30–31 | | `0x04` | 1 | Row | | `0x05` | 3 | Page | -| `0x08` | 4 | reserved | +| `0x08` | 4 | **Chain stamp** — must equal the first chain page's header `0x08`. Non-zero only on the multi-page form; zero on inline and single-page | Flags (byte `0x03` masked with `0xC0`; its low six bits belong to the length): - `0x80` **inline** — the payload follows the descriptor in the row. -- `0x40` **single LVAL page** — the row at (page, row) *is* the whole payload. +- `0x40` **single LVAL page** — the row at (page, row) *is* the whole payload. Several such values **share** + a page; deleting one retires its row to a 0-length deleted + overflow tombstone and re-lays the page, and + the page is released as type `0x09` once the last of them is gone + ([page-05 §9](page-05-usage-maps.md)). - `0x00` **multi-page** — the payload is chained across LVAL pages; each chunk's row begins with a 4-byte pointer (`[row:1][page:3]`) to the next chunk (zero on the last), followed by chunk data. Each chunk row is **`MAX_LONG_VALUE_ROW_SIZE` = 4076 bytes** (Jet4; Jet3 = 2032) — a 4-byte pointer + up to 4072 data bytes — except the last, which is shorter. Verified against ACE's own - chained OLE (Northwind Employee photos: 4076, 4076, 2606-byte chunk rows). + chained OLE (chunk rows of 4076, 4076 and 2606 bytes). + +> **The chain stamp binds a descriptor to its first chain page, and ACE enforces it.** A multi-page +> descriptor's `0x08` and the header `0x08` of the **first** page of its chain hold the same four bytes; +> later chunk pages, single-page (`0x40`) LVAL pages, and ordinary data pages all hold zero. Only the first +> page is bound: patching every *later* chunk page's `0x08` while leaving the descriptor and the first page +> alone changes nothing ACE notices, and patching the first page alone is refused. Which is what the check +> is for — the descriptor is the only way into the chain that a stale pointer can arrive by, since every +> chunk after it is reached from a page already validated. ACE writes +> `GetTickCount()` there — milliseconds since the writing machine booted, so it is machine- and +> boot-relative and reproduces nowhere, like the database creation date. **The value is arbitrary; only the +> agreement matters.** In an ACE-written file, setting *both* copies to `DEADBEEF` reads back fine, setting +> *both* to zero reads back fine, and changing *either one alone* — to any value, zero included — makes ACE +> refuse the record. It refuses it as *"you and another user are attempting to change the same data at the +> same time"*, the same misleading concurrency message an +> oversized record gets, and `CompactDatabase` does not repair such a row: it drops the value. +> +> **The tag is minted per write of the chain**, not per write of the row: inserting stamps both copies, +> rewriting the value writes a new chain and a fresh stamp in both, updating another column of the same row +> leaves it untouched, and a value that shrinks to the single-page form drops both to zero. The check also +> fires only when the long value is **materialised** — ACE will happily `UPDATE` another column of a row +> whose stamp disagrees, and refuse the moment anything reads the memo. +> +> That reading also explains the choice of clock. A chain's pages can be freed and reused by a later value, +> and a stale descriptor would then point at a page holding someone else's data; a stamp that differs +> between successive uses of the same page catches exactly that, and needs to be distinct rather than +> meaningful. **LibRed writes the same stamp in both places and checks it on read**, refusing a chain whose +> entry page disagrees with the descriptor that reached it. Like the database creation date, the value does +> not reproduce between two runs and is not expected to. LibRed is single-writer and so cannot produce the +> interleaving ACE guards against, but it can be handed a file another engine wrote — and the guard is +> groundwork for multi-user concurrency, where this is exactly the check that has to exist. ACE accepts an OLE/binary payload of `0x3FFFFFFF` bytes (1 GiB − 1) and rejects `0x40000000`. That is a **byte** limit, so the Memo **character** limit is it divided by the two bytes a character costs: Jet 4 stores text as UTF-16LE, and compressed Unicode (§7) is an optimisation on top of that rather than a -different encoding (`LongTextStorageAccessTests`). +different encoding. -Both ends measured directly rather than inferred, with ACE authoring and both engines verifying every -character: +Both ends measured, with ACE authoring and both engines verifying every character: | | bytes | outcome | | --- | ---: | --- | @@ -49,10 +81,8 @@ the *storage form*, and at this scale the answer is always no, because a chained (see below) and anything near the ceiling is chained many times over. So "how many characters fit" is the byte ceiling divided by two, whatever the text holds. -Worth measuring anyway, because the alternative was cheap to believe: if compression had applied, an -all-ASCII Memo would have reached twice as far. It does not — ASCII and non-ASCII agree character for -character at both ends, 536,870,911 accepted and 536,870,912 refused, and the accepted one takes the same -~2 hours to write. Confirmation of the model rather than a surprise in it. +An all-ASCII Memo therefore does **not** reach twice as far: ASCII and non-ASCII agree character for +character at both ends, 536,870,911 accepted and 536,870,912 refused. > **The rejection is not cheap and not lossy.** ACE took ~109 minutes to refuse the over-long Memo — it does > not pre-check the declared length, it processes the whole value and fails at the end — and the database @@ -69,7 +99,7 @@ character at both ends, 536,870,911 accepted and 536,870,912 refused, and the ac > (`M MEMO WITH COMP`) and does set the capable flag, but compression is decided *after* the storage form, > and a **chained** value is never compressed. Anything near the byte ceiling is chained by a wide margin, > so the character limit is unaffected however the column was declared. LibRed implements the attribute and -> matches ACE's choice byte-for-byte (`CompressedTextAccessTests`); the eligibility rules are in +> matches ACE's choice byte-for-byte; the eligibility rules are in > [data-types.md §7](data-types.md#7-compressed-unicode). **Choosing the storage form.** All three forms are chosen on the value's **uncompressed** UTF-16 length: @@ -77,14 +107,14 @@ character at both ends, 536,870,911 accepted and 536,870,912 refused, and the ac | uncompressed length | form | flag | | --- | --- | --- | | ≤ 64 bytes | inline, payload follows the descriptor | `0x80` | -| 66 … 3816 bytes | one LVAL page | `0x40` | -| > 3816 bytes | chained across LVAL pages | `0x00` | +| 65 … 3816 bytes | one LVAL page | `0x40` | +| ≥ 3817 bytes | chained across LVAL pages | `0x00` | -> **3816 is not the same number as the 4076-byte chunk row**, and conflating them was a real bug: LibRed used -> 4076 as its single-page threshold and so kept 3818–4076 byte values on one page where ACE chains them. -> Measured both ways — a plain `LONGCHAR` and a `WITH COMP` one behave identically, 1908 characters (3816 -> bytes) staying single-page and 1909 (3818) chaining. What fixes the boundary at 3816, rather than the 4076 -> a row can actually hold, is **not established**; the ~260-byte margin is unexplained. +> **3816 is not the same number as the 4076-byte chunk row**: a writer that uses 4076 as its single-page +> threshold keeps 3817–4076 byte values on one page where ACE chains them. A plain `LONGCHAR` and a +> `WITH COMP` one behave identically, 1908 characters (3816 bytes) staying single-page and 1909 (3818) +> chaining. What fixes the boundary at 3816, rather than the 4076 a row can actually hold, is **not +> established**; the ~260-byte margin is unexplained. LVAL pages are data pages (type `0x01`) whose owner field (`0x04`) is the ASCII marker `LVAL`. @@ -95,10 +125,9 @@ LVAL pages are data pages (type `0x01`) whose owner field (`0x04`) is the ASCII > a `(page,row)`, terminate at zero exactly when the declared length is reached, and neither underfill nor > overrun that length. Before reclaiming a replaced chain, LibRed validates the complete chain and requires > every page to be present in that column's owned-pages map; only then does it begin clearing maps/free bits. -> Those subsequent writes are atomic whenever a transaction is open — the page-level undo log exists now -> (`docs/design/transactions.md`), and the engine opens one per statement — so a failed reclamation rolls -> back with the statement. A direct `LibRed.Core` caller that opens none gets the same non-atomic behaviour -> as any other multi-page write. +> Those subsequent writes are atomic whenever a transaction is open (`docs/design/transactions.md`), and the +> engine opens one per statement, so a failed reclamation rolls back with the statement. A direct +> `LibRed.Core` caller that opens none gets the same non-atomic behaviour as any other multi-page write. ### 3.3.2 Column usage-map list (trailing the index names) @@ -119,37 +148,49 @@ list (terminator included) counts toward the definition, not free space. > OLE (`0x0B`)** columns appear here — *not* plain **Text (`0x0A`)**, even though Text is > variable-length — because only memo/OLE have their own long-value (LVAL) page chains that need > usage maps; Text is stored inline in the row. Verified by correlating each entry with its column -> type: Categories → `{col2 Memo, col3 OLE}`, Employees → `{col14 OLE, col15 Memo}`, Suppliers → -> `{col11 Memo}`, and — the clincher — **Customers, with 11 Text columns and no memo/OLE, has an +> type: every entry is a Memo or OLE column, and **a table with Text columns but no memo/OLE has an > empty list**. So mdbtools' name is imprecise; the list is keyed to long-value columns. +### Dropping a long-value column + +`DROP COLUMN` of a memo or OLE column is a metadata edit like any other drop ([page-02a](page-02a-tdef.md)) — +existing rows keep the column's bytes, now dead, and its LVAL pages keep their contents — plus three steps for +the long values: + +1. its §3.3.2 entry leaves the definition; the other long-value columns keep theirs, and their map records + keep their row numbers; +2. its owned-map and free-map records are retired from their holder exactly as `DROP TABLE` retires a + long-value column's ([page-05 §9](page-05-usage-maps.md)): the owned pages' bits cleared except the page + still in its free map, then each row tombstoned, the rows below sliding up; +3. every page in its owned map — single-value pages and chain pages alike — goes back to the global free-pages + map at close. + ## Writing long values -> **Writing.** LibRed inlines a memo/OLE value only up to **64 bytes** (Jackcess -> `MAX_INLINE_LONG_VALUE_SIZE`, same for Jet3/Jet4): the 12-byte descriptor with length + the `0x80` -> flag (bytes `0x04`–`0x0B` zero) then the payload (memo = UTF-16LE, OLE = raw bytes). A value **larger -> than 64 bytes** is written to its own **single LVAL page** (`0x40` descriptor, `LongValueWriter`) — -> `RowInserter` materialises it before encoding. This matters for Access, not just LibRed: Access +> **Writing.** LibRed inlines a memo/OLE value only up to **64 bytes** (same for Jet3/Jet4): the 12-byte +> descriptor with length + the `0x80` flag (bytes `0x04`–`0x0B` zero) then the payload (memo = UTF-16LE, +> OLE = raw bytes). A value of **65–3816 bytes** is written as one row on an **LVAL page** (`0x40` +> descriptor, `LongValueWriter`; rows share a page, see below) — `RowInserter` materialises it before +> encoding. This matters for Access, not just LibRed: Access > tolerates an inline value its reader resolves, but **rejects an over-64-byte value inlined** (e.g. it > opens the database yet fails to *run* a view whose subquery `Expression` was inlined; on an LVAL page -> it runs — verified against the derived-table view, §11). A value **larger than one LVAL row** (4076 -> bytes) is written as a **chain** (`0x00` descriptor): the payload is split into 4072-byte data chunks, +> it runs — verified against the derived-table view, §11). A value of **3817 bytes or more** is written as +> a **chain** (`0x00` descriptor): the payload is split into 4072-byte data chunks, > each on its own page with a 4-byte next-pointer, matching ACE byte-for-byte (verified: LibRed and > Access both read back memo values from 65 bytes to 100 KB — single-page and multi-page). > > **LibRed writes the §3.3.2 entry + empty usage maps for every memo/OLE column** — byte-faithful with > ACE, whose usage-map page lays the records out as: row 0 table-owned, row 1 table-free, then one row -> **per index**, then two rows (owned/free) **per long-value column** (verified against Northwind -> Categories and against an ACE-created 80-memo-column table). +> **per index**, then two rows (owned/free) **per long-value column** (verified against ACE-created +> tables). > > **That order is the DDL's, not a fixed rule — ACE assigns the rows in declaration order.** An *inline* > `PRIMARY KEY` is declared before the long-value columns and takes row 2, giving the layout above; a > trailing `CONSTRAINT pk PRIMARY KEY (…)` clause is created *after* them, so on a two-memo table ACE gives -> the columns rows 2–5 and the index row 6. Measured both ways at 1, 2, 5, 26, 27, 28 and 40 long-value -> columns. LibRed always writes the inline order — the declaration position is lost between the parser and -> `CreateTable` — so it matches ACE byte-for-byte for inline keys and differs by the row numbering alone for -> a named constraint. Both files are self-consistent and ACE reads either. -> `TdefByteParityAccessTests.Usage_map_rows_follow_declaration_order` holds both measurements. +> the columns rows 2–5 and the index row 6. Measured both ways. LibRed always writes the inline order — the +> declaration position is lost between the parser and `CreateTable` — so it matches ACE byte-for-byte for +> inline keys and differs by the row numbering alone for a named constraint. Both files are self-consistent +> and ACE reads either. > > The spill rule applies here too, and to whichever comes last: at 27 memo columns the index still fits the > primary page (row 56, the 57th record), and at 28 it goes to **row 0 of a page of its own** — the same @@ -158,23 +199,22 @@ list (terminator included) counts toward the definition, not free space. > fit all its used/free maps on one page. Access fills the primary page (data + indexes + as many *whole* > columns as fit — 27 columns alongside a single index), then gives **each remaining long-value column its > own dedicated usage-map page** with owned = row 0, free = row 1. LibRed reproduces this exactly (verified: -> an 80-memo table lands 27 columns on the primary page at rows 3–56, then one page each for the rest; -> ACE opens it and round-trips an 8000-char value written to an overflow column). **`CREATE INDEX` on such -> a table spills too**: with the primary page full, ACE does not compact or reuse it but allocates a page -> holding the new index's map alone, at row 0 (verified on a 40-memo table: after `CREATE INDEX` the -> primary page still has its 57 rows and the new index block's `+0x22` pointer reads row 0 of a fresh -> page). Only when the primary page still has room for another 69-byte record does the new index's map go -> there, appended after the existing rows. Each column's §3.3.2 -> `used_pages`/`free_pages` pointers, and the index blocks' `+0x22` pointers, carry the resolved (row, page). -> For a fresh table all these maps are empty. When LibRed writes a value to an LVAL page -> (§8), it now **sets that page's bit in the column's owned-pages *and* free-pages maps** — both §3.3.2 +> 27 columns land on the primary page at rows 3–56, then one page each for the rest; ACE opens the table and +> round-trips a value written to an overflow column). **`CREATE INDEX` on such a table spills too**: with +> the primary page full, ACE does not compact or reuse it but allocates a page holding the new index's map +> alone, at row 0 (verified: after `CREATE INDEX` the primary page still has its 57 rows and the new index +> block's `+0x22` pointer reads row 0 of a fresh page). Only when the primary page still has room for +> another 69-byte record does the new index's map go there, appended after the existing rows. Each column's +> §3.3.2 `used_pages`/`free_pages` pointers, and the index blocks' `+0x22` pointers, carry the resolved +> (row, page). For a fresh table all these maps are empty. When LibRed writes a value to an LVAL page (§8), +> it **sets that page's bit in the column's owned-pages *and* free-pages maps** — both §3.3.2 > pointers are parsed from the TDEF (`TableDefinitionPage.LongValueOwnedMaps` / `LongValueFreeMaps`, keyed > by column id) and the inline bitmap bit is set. **Pages are packed like Access:** a value up to one row > is appended to the first **free-map** page with room (many small values share a page as separate rows); > only when none has room is a fresh page allocated (owned + free). A page is dropped from the free map > once it can't hold the smallest long value (65-byte payload + its 2-byte slot). This reproduces Access's -> layout — MSysQueries.Expression **owns** {42, 282} but **frees** only {282}, the current append target; -> and 20 medium memos land on ~2 pages (full one owned-only, current one owned+free), not 20. The same +> layout — a column **owns** every page it has filled but **frees** only the current append target, so +> medium memos share a few pages (full ones owned-only, the current one owned+free), not one each. The same > packing is used for the MSysObjects **LvProp** property blob (via `RowInserter.StorePackedLongValue`) — > but always to a page, never inline (Access reads object properties only from a page), so two tables' > DEFAULT/CHECK blobs share one LvProp page. A chained value uses dedicated pages. A page outside the inline @@ -185,12 +225,11 @@ and `MapPages` reads either form back. > **The terminating `0xFFFF` is mandatory on write — even for a table with no long-value > columns** (where the list is empty and the `0xFFFF` is the only bytes here). Omitting it makes > Access reject the whole table with *"Unrecognized database format"* even though every other byte -> of the TDEF is valid — verified by byte-diffing an ACE-created single-index table against a LibRed -> one whose only difference was the missing terminator. LibRed's reader doesn't consume this list -> (it stops after the named indexes; long values are located via the in-row LVAL pointer, not these -> maps), but the terminator **must be written**. A table with memo/OLE columns must additionally -> allocate the usage-map records and emit a real `{col_num, used, free}` entry per long-value column -> — verified against Northwind's Categories (cols 2/3) and Employees (cols 14/15). +> of the TDEF is valid — verified by byte diff against an ACE-created table. LibRed's reader doesn't +> consume this list (it stops after the named indexes; long values are located via the in-row LVAL +> pointer, not these maps), but the terminator **must be written**. A table with memo/OLE columns must +> additionally allocate the usage-map records and emit a real `{col_num, used, free}` entry per long-value +> column — verified against ACE-authored tables. > > The §3.3.2 entry is only strictly *required* once a value spills to LVAL pages — an entry-less table > still round-trips inline values through both LibRed and Access, but Access fails *"Not a valid bookmark"* diff --git a/src/LibRed/docs/format/page-00-database.md b/src/LibRed/docs/format/page-00-database.md index 936890782..07d66f87f 100644 --- a/src/LibRed/docs/format/page-00-database.md +++ b/src/LibRed/docs/format/page-00-database.md @@ -11,16 +11,17 @@ | `0x04` | 15 | Format identifier ASCII: `Standard Jet DB` or `Standard ACE DB` | | `0x13` | 1 | NUL terminator of the identifier string | | `0x14` | 1 | Version byte (see below). mdbtools reads `jet_version` as a 4-byte word at `0x14`; the version is its low byte | -| `0x15` | 1 | Version **minor/update** byte: **`0x01` on ACE 14 / Access 2010 (version `0x03`)**, `0x00` on every other version tested (Jet 4, ACE 12/17). mdbtools says this is always zero — not universally true. Purpose beyond distinguishing the 2010 format unknown. **An in-place version raise moves `0x14` only**, so raising a 2010 file leaves the pair `(0x05, 0x01)`, which no ACE-authored file carries — measured, and **ACE opens the result and reads and writes it normally** (`AuditRegressionAccessTests`), so the pair is tolerated | +| `0x15` | 1 | Version **minor** byte: **`0x01` on a database created in the 2010 format (version `0x03`)**, `0x00` when created in any other. **A version raise writes `0x00`** whatever the target — including a raise *onto* `0x03`. Purpose otherwise unknown | | `0x16` | 2 | Unknown (zero observed) | | `0x18`–`0x98` | 128 | **Obfuscated header** — XOR'd with a fixed 128-byte mask (§2.1). Jet 3 masks 126 bytes. Fields below are offsets into it. | -| `0x18`, `0x1C` | 4+4 | Fixed constants `0x00000100`, `0x00000101` (not page pointers — out of range in small files) | +| `0x18` | 4 | **Global free-pages map pointer** — `[row:1][page:3]`; `0x00000100` = page 1 row 0 in every file ACE writes ([page-05 §9.1](page-05-usage-maps.md)) | +| `0x1C` | 4 | **Global released-pages map pointer** — `[row:1][page:3]`; `0x00000101` = page 1 row 1 ([page-05 §9.1](page-05-usage-maps.md)) | | `0x20`–`0x2C` | 4×4 | **System-catalog bootstrap pointers**: TDEF pages of `MSysObjects` / `MSysACEs` / `MSysQueries` / `MSysRelationships` = `2, 3, 4, 5`. `0x20` is the **catalog root** (how the engine finds `MSysObjects`). | | `0x30`–`0x3B` | 12 | Reserved (zero) | | `0x3C` | 2 | **ANSI code page** — LE (`0x04E4` = 1252, `0x04E2` = 1250) | | `0x3E` | 4 | **Database (encryption) key** — 0 when there is no password | | `0x42` | 40 | **Password** (Jet 4; Jet 3 = 20 bytes) — additionally masked by a creation-date-derived value, so an empty password does not read as zeroes | -| `0x6A` | 4 | Fixed constant `0x000011A6` — invariant across the entire Jet 4 lineage (every version/engine/collation/language tested); likely a validation sentinel/marker (cf. the `0x0659` TDEF record marker, §3.1), exact purpose unconfirmed | +| `0x6A` | 4 | **Creating engine's build number** — `0x000011A6` (4518) on everything ACE writes, but **not a constant**: Jet-4-authored files carry the build of the `msjet40.dll` that created them (see below) | | `0x6E` | 4 | **Default text collating sort order** — a 32-bit LCID with the version in its unused top byte: LANGID (2, LE, `0x0409` = 1033 en-US), **sort id** at `0x70`, **sort-order version** at `0x71` (0 = legacy table, 1 = the Access-2010 order). Byte-for-byte the same layout as a column descriptor's `0x0B`–`0x0E` | | `0x72` | 8 | **Database creation timestamp** — OLE automation `double` (days from 1899-12-30) | | `0x98` | 4 | **Past the masked window** (cleartext). Fixed constant `0x00000654` (1620), undecoded | @@ -39,7 +40,7 @@ Version byte → format: | `0x01` | Jet 4 (Access 2000–2003) | 4096 | MDB | | `0x02` | ACE 12 (Access 2007) | 4096 | ACCDB | | `0x03` | ACE 14 (Access 2010) | 4096 | ACCDB | -| `0x04` | ACE 15 (Access 2013) — **reserved, never emitted** | 4096 | ACCDB | +| `0x04` | ACE 15 (Access 2013) — **never emitted, and ACE refuses to open a file carrying it** | 4096 | ACCDB | | `0x05` | ACE 16 (Access 2016) — triggered by **Large Number** | 4096 | ACCDB | | `0x06` | ACE 17 (Access 2019+) — triggered by **Date/Time Extended** | 4096 | ACCDB | @@ -53,33 +54,34 @@ The version byte is **"the minimum ACE engine whose format features this file us byte per engine release whether or not that release adds a format-forcing feature. The only features that push past `0x03` are the two new *data types*: **Large Number** (Int64) → `0x05`, and **Date/Time Extended** (datetime2) → `0x06`. Access **2010 through 2019 all default to `0x03`** unless a file actually uses one of those types. -**`0x04` (ACE 15 / Access 2013) is reserved but never stamped** — 2013 added no format-forcing data type, so its -files fall back to `0x03` (verified: a real `db2013` reads `0x03`; jackcess ships no 2013 fixture; Access 2013 -defaults to the 2007-2016 format). LibRed maps `0x04` to the `0x03` (2010) layout rather than a clone class. +**`0x04` (ACE 15 / Access 2013) is never stamped** — 2013 added no format-forcing data type, so its +files fall back to `0x03` (verified; Access 2013 defaults to the 2007-2016 format). LibRed maps `0x04` to the +`0x03` (2010) layout rather than a clone class. + +> **ACE does not merely avoid `0x04` — it refuses it.** An otherwise well-formed, *empty* database carrying the +> byte cannot be opened by any provider; restamping `0x14` to `0x03` makes the identical bytes open. +> +> The format at `0x04` is otherwise exactly `0x03`, so LibRed is deliberately **asymmetric**: `FromVersionByte` +> accepts the byte and reads the 2010 layout, while `DatabaseCreator.CreateEmpty` refuses to write it and points +> the caller at `Version14_2010`. + A genuinely **unknown** version byte on an `.accdb` that still carries the cleartext `"4.0"` engine string at `0x9C` is read as the **latest known ACE** layout (currently ACE 17) — the format grows conservatively, so an unrecognised byte is almost certainly a newer 4KB ACE variant; the `"4.0"` guard stops a genuinely different future engine (e.g. a `"5.0"` string) from being mis-read as ACE. -**Upgrading an existing file is that byte and nothing else** (verified 2026-08-26 against ACE over OLE DB, -from a DAO-created ACE 12 baseline — `dbVersion120`, version `0x02`). Adding a `DATETIME2` column through ACE -changes exactly one byte of page 0: `0x14`, `0x02` → `0x06`. A control arm adding an ordinary `DATETIME` column -to the same baseline is what isolates it — the only other byte either arm touched was the opening user's -commit slot at `0xE02` (§2.2), which moves for any write at all. - The byte is **sufficient, not merely necessary**: writing `0x06` to `0x14` by hand upgrades an ACE 12 file in place. ACE then opens it, data written before the flip is still readable, and `ALTER TABLE … ADD COLUMN … DATETIME2`, `INSERT`, `SELECT` and `CREATE TABLE` with the type all work — ACE adding nothing further to page 0 -of its own. Guard: `AceDateTime2UpgradeTests`. ACE's DDL accepts only the bare spelling **`DATETIME2`**; -`DATETIME2(7)`, `DATETIMEEXTENDED`, `DATE/TIME EXTENDED` and `DATETIMEOFFSET` are all syntax errors. +of its own. ACE's DDL accepts only the bare spelling **`DATETIME2`**; `DATETIME2(7)`, `DATETIMEEXTENDED`, +`DATE/TIME EXTENDED` and `DATETIMEOFFSET` are all syntax errors. -The `0x05` / **Large Number** route behaves the same way and is now measured too (verified 2026-08-26): a -`CREATE TABLE … BIGINT` issued through ACE against an ACE 12 file moves `0x14` from `0x02` to **`0x05`** — not -to `0x06`, confirming the two types really do sit at different formats. Guard: -`BigIntKeyEncodingTests.Adding_a_bigint_column_makes_ace_raise_the_file_to_ace16`. +The `0x05` / **Large Number** route behaves the same way (verified): a `CREATE TABLE … BIGINT` issued through +ACE against an ACE 12 file moves `0x14` from `0x02` to **`0x05`** — not to `0x06`; the two types sit at +different formats. **LibRed performs this upgrade itself**, as ACE does: DDL introducing a type the open file is too old for -raises the version byte instead of refusing (`StatementExecutor.MapColumn` → +raises the version byte, and clears the minor, instead of refusing (`StatementExecutor.MapColumn` → `JetDatabase.EnsureFormatAtLeast` → `PageChannel.RaiseFormatVersion`). Three properties are worth recording, because each is a place the obvious implementation goes wrong: @@ -93,8 +95,14 @@ because each is a place the obvious implementation goes wrong: That is inherent — the column it would find is one it could not read either. Verified against the real engine: ACE opens a file LibRed upgraded in place and reads the value that forced -the upgrade (`DateTime2CreatedDatabaseAccessTests`). A saved query's *parameter* type is deliberately excluded -— it declares no storage, and what ACE does with a new-type parameter in `MSysQueries` has not been probed. +the upgrade. A saved query's *parameter* type is deliberately excluded — it declares no storage, and what ACE does with a new-type parameter in `MSysQueries` has not been probed. + +**Allocation bootstrap.** `0x18` and `0x1C` are the two things an engine needs before it can allocate a page, +and so cannot look up through the catalog: record pointers to the global free-pages map and the global +released-pages map. They are `[row:1][page:3]` pointers, like a TDEF's usage-map pointers, because a usage +map is a record on a data page; the catalog pointers that follow are plain page numbers because a table +definition is a page. What the maps hold, and how ACE validates and follows the pointers, is +[page-05 §9.1](page-05-usage-maps.md). **Catalog bootstrap.** Reading the database is a two-step hop from page 0: the pointer at `0x20` gives the `MSysObjects` TDEF page (2), and `MSysObjects` then lists every other object (each table's row `Id` is *its* @@ -105,13 +113,13 @@ TDEF page). LibRed reads `0x20` into `DatabaseDefinitionPage.CatalogRootPage` an > **Creation from scratch (implemented — `DatabaseCreator`).** A minimal bootable page 0 needs the mask, the > code page / collation / creation date, and this pointer block aimed at the four core system tables > (`MSysObjects`, `MSysACEs`, `MSysQueries`, `MSysRelationships`) — the minimum catalog a new file must -> contain. LibRed now synthesises all of this natively (no DAO/ADOX, no template copy) and the result opens -> **clean in the Access desktop GUI** (no permission popups, no auto-compact error). Two non-obvious facts made -> that work, both recorded below: the **creation date is bound to the on-disk security SIDs** (§2.3), and the -> file must **not** hand-create the `MSysAccessStorage` / `MSysNavPane*` tables — real DAO files omit them and -> Access adds them (with the nav-pane long SID) on first open (verified across ~135 pure-DAO files). +> contain. LibRed synthesises all of this natively (no DAO/ADOX, no template copy) and the result opens +> **clean in the Access desktop GUI** (no permission popups, no auto-compact error). Two requirements are +> non-obvious: the **creation date is bound to the on-disk security SIDs** (§2.3), and the file must **not** +> hand-create the `MSysAccessStorage` / `MSysNavPane*` tables — real DAO files omit them and Access adds them +> (with the nav-pane long SID) on first open (verified). -> **How the reference engine lays out a new file** (DAO-created ACE 12, 42 pages — `DaoPageLayoutProbeTest`). +> **How the reference engine lays out a new file** (DAO-created ACE 12, 42 pages). > Per table the allocation order is **TDEF → usage-map page → one page per index root**, in table-creation > order; both usage maps share one page (owned = row 0, free = row 1, inline), which is what every TDEF's > `0x37`/`0x3B` pointers show. **Data pages are allocated lazily on first insert**, so they appear out of @@ -119,7 +127,7 @@ TDEF page). LibRed reads `0x20` into `DatabaseDefinitionPage.CatalogRootPage` an > > | pages | contents | > | --- | --- | -> | `0`, `1` | database definition; global free-pages map | +> | `0`, `1` | database definition; the global free-pages (row 0) and released-pages (row 1) maps | > | `2`–`5` | the four core TDEFs — fixed, because page 0's bootstrap pointers name them | > | `6`, `9`, `11`, `13` | usage maps for MSysObjects / MSysACEs / MSysQueries / MSysRelationships | > | `7`, `8`, `10`, `12`, `14`–`16` | their index roots (2 + 1 + 1 + 3), each a leaf page | @@ -136,6 +144,27 @@ TDEF page). LibRed reads `0x20` into `DatabaseDefinitionPage.CatalogRootPage` an > `MSysComplexType_Attachment.FileURL` is the last descriptor with id 0. Sorting is by name, not > fixed-before-variable: `ColumnName` is variable-length and still sorts first. +### `0x6A` — the creating engine's build number + +The build number of the engine that created the file, stamped once and preserved across copy and compact. +Observed values: + +| value | engine build | +| --- | --- | +| `0x000011A6` (4518) | ACE `12.0.4518` — Office 2007 RTM | +| `0x0000261C` (9756) | Jet `4.0.9756` | +| `0x000021AA` (8618) | Jet `4.0.8618` | +| `0x00000B6F` (2927) | Jet `4.0.2927` — SP3 | +| `0x000009D9` (2521) | Jet `4.0.2521` — RTM | + +**ACE writes 4518 unconditionally** — every ACE version (12/14/16/17) stamps its own RTM build, including when +writing a Jet 4 `.mdb`. A Jet 4 build appears only in a file a real `msjet40.dll` created, and is that engine's +service-pack build. + +> That the value *is* an engine build is **inferred** — from the observed values coinciding with shipped build +> numbers — not measured. That the field varies, and varies only across Jet 4 files of differing vintage, is +> measured. LibRed writes 4518. + ### 2.1 The obfuscated header (`0x18`–`0x98`) From `0x18` for **128 bytes** (Jet 4 / ACE; 126 for Jet 3), page 0 is obfuscated by XOR-ing the @@ -158,19 +187,11 @@ DF B1 77 F4 13 43 CF AF B1 33 34 61 79 5B 92 B5 ; 0x58 CF 65 ED FF 07 C7 46 A1 78 16 0C ED E9 2D 62 D4 ; 0x88 ``` -**Verification (why this is recorded despite being an external mask).** The mask is Jackcess's -`BASE_HEADER_MASK`, but it is **not adopted on faith** — it is confirmed against real files two ways: - -1. **Reproduces bytes recovered from first principles.** Independently, by a known-plaintext attack — - varying one Access setting and reading its plaintext from an unobfuscated in-file copy — LibRed - recovered the mask at three fields: the code page (`mask[0x3C]=7B,42` — de-obfuscation yields the - canonical Windows code pages `0x04E4`/`0x04E2`), the collation LCID (`mask[0x6E]=01,1B`, checked - against each column descriptor's own locale at `0x0B`–`0x0C` over five distinct LCIDs), and the - creation date (`mask[0x72]=12 4F 4A 94 6C 3E 60 26`, matching `MSysObjects.DateCreate` to the - second). The Jackcess mask matches all twelve of those bytes exactly. -2. **Decodes every fixture sensibly.** Applied whole, it yields valid code pages (1252/1250), the - expected LCIDs, correct creation dates, a zero database key (no-password files), and an empty - password that unmasks to the creation-date-derived pattern (below). +**Verified** against real files: at the code page (`mask[0x3C]=7B,42`) the mask yields the canonical +Windows code pages `0x04E4`/`0x04E2`; at the collation LCID (`mask[0x6E]=01,1B`) it agrees with each +column descriptor's own locale at `0x0B`–`0x0C`; at the creation date (`mask[0x72]=12 4F 4A 94 6C 3E 60 26`) +it matches `MSysObjects.DateCreate` to the second. Applied whole, it also yields a zero database key on +no-password files and an empty password that unmasks to the creation-date-derived pattern (below). **Decoded fields** (all little-endian; `DatabaseDefinitionPage` → `JetDatabase`): @@ -184,23 +205,16 @@ CF 65 ED FF 07 C7 46 A1 78 16 0C ED E9 2D 62 D4 ; 0x88 - **Password (`0x42`, 40 bytes)** — *not* decoded to a value, and the two families differ: - **Jet 4 `.mdb`**: light access-control obfuscation only — the field is the password XOR the base mask XOR an **additional 4-byte mask = `(int)creationDate`** (repeated). An empty password - therefore unmasks to that creation-date pattern, not zeroes (this is the per-file variation once - mistaken for a signature — there is no ESE-style machine signature here). The plaintext is - recoverable, as mdbtools/Jackcess do. **`(int)creationDate` = the 8-byte creation-date double at - `0x72` truncated to a 32-bit int, written little-endian and cycled over the 40-byte field** (matching - jackcess `getPasswordMask`; the high 2 bytes are usually 0, so half the field is plaintext UTF-16). + therefore unmasks to that creation-date pattern, not zeroes — the per-file variation is this mask, + not a signature (there is no ESE-style machine signature here). The plaintext is recoverable, as + mdbtools does. **`(int)creationDate` = the 8-byte creation-date double at `0x72` truncated to a 32-bit + int, written little-endian and cycled over the 40-byte field** (the high 2 bytes are usually 0, so half + the field is plaintext UTF-16). The mask depends on the creation date alone, not on the password. **Setting/removing this password is implemented** (`DatabaseEncryption.SetJetPassword` / `RemoveJetPassword`): write `UTF-16LE(password)` zero-padded to 40 bytes, XOR the date mask, then the - base header mask — the exact inverse of the read. **Verified byte-identical to Access's own output**: - `SetJetPassword` on a copy of `2002plain.mdb` reproduces Access-set `Test1`/`Test2`/`AAAA`/`z` files - bit-for-bit in the `0x42` field (`LegacyJetPasswordTests.SetJetPassword_matches_access_output`). Those - fixtures are not committed, so that case **skips with a reason** unless they are present — point - `LIBRED_ENCTEST_DIR` at them to run it. The rest of `LegacyJetPasswordTests` builds its own Jet 4 header and - covers the field transformation, limits, removal, and encoding independence on every platform. This is - password-only obfuscation — the - data pages stay plaintext (`0x3E` key = 0); it is a *different* feature from Jet RC4 page encryption - (§2.4), which the "Encode/Encrypt" menu applies. The earlier "per-file SID mask = f(date,password)" - theory was a misdiagnosis — the mask is simply `(int)creationDate`. + base header mask — the exact inverse of the read. **Verified byte-identical to Access's own output** in + the `0x42` field. This is password-only obfuscation — the data pages stay plaintext (`0x3E` key = 0); + it is a *different* feature from Jet RC4 page encryption (§2.4), which the "Encode/Encrypt" menu applies. - **ACE `.accdb`**: real encryption — this region is an encryption **verifier**, not recoverable plaintext (an actual password decodes to random-looking bytes under the Jet 4 scheme). Recovering it is a crypto attack, not format work. @@ -218,16 +232,13 @@ CF 65 ED FF 07 C7 46 A1 78 16 0C ED E9 2D 62 D4 ; 0x88 column. The block's semantics — what a non-zero sort id means, which version is which — belong to that descriptor field and are [page-02b §3.4](page-02b-columns.md). - **Creation date (`0x72`, 8 bytes)** → `CreationDate` — an OLE `double`. Matches the earliest - `MSysObjects.DateCreate`; on an *edited* database (e.g. Northwind) it is the **file's** creation - instant and can differ from the first object's by minutes. **Unlike a normal Jet/ACE `DateTime` - column (whole-second resolution), this header stamp carries sub-second precision** — verified across - 144 files, every value sits a whole number of milliseconds off a whole second (−284, −383, +78, +462 ms…), - i.e. ~1 ms resolution, consistent with a Windows `SYSTEMTIME`. The column codec truncates to seconds; - this field is written straight from the OS clock and keeps the milliseconds. See §2.3 — those low bits - matter because the security SIDs are bound to them. - -Regression tests: `DatabaseDefinitionPageTests.Decodes_creation_date_matching_catalog` and -`Decodes_code_page_and_default_collation`. + `MSysObjects.DateCreate`; on an *edited* database it is the **file's** creation instant and can + differ from the first object's by minutes. **Unlike a normal Jet/ACE `DateTime` column (whole-second + resolution), this header stamp carries sub-second precision** — verified: every value sits a whole + number of milliseconds off a whole second (−284, −383, +78, +462 ms…), i.e. ~1 ms resolution, + consistent with a Windows `SYSTEMTIME`. The column codec truncates to seconds; this field is written + straight from the OS clock and keeps the milliseconds. See §2.3 — those low bits matter because the + security SIDs are bound to them. ### 2.2 The user commit-byte table (`0xE00`–`0xFFF`) @@ -247,12 +258,11 @@ page-level read/write registration. **`00 00` means "mid-write to disk"**, and ` "accessed a corrupted page" — either one *without a matching user lock* makes Jet declare the database suspect and demand a repair before it will open. -#### The slot is a little-endian commit counter (verified 2026-08-26) +#### The slot is a little-endian commit counter (verified) Above those low reserved values, a slot is **one 16-bit little-endian counter of that user's committed -writes** — not an enumerated state, which is how this file previously described it and how the note in -[page-05](page-05-usage-maps.md) half-described it ("bumps a counter … not yet decoded"). Measured against -`Microsoft.ACE.OLEDB.16.0` on one connection, watching `0xE02` (slot 1, the first shared user): +writes** — not an enumerated state. Slot 1 (`0xE02`, the first shared user) across a sequence of +statements on one ACE connection: | | slot 1 | as LE16 | | --- | --- | --- | @@ -265,7 +275,7 @@ writes** — not an enumerated state, which is how this file previously describe | +40 inserts | `01 03` | 769 — exactly +40 | | +240 more | `F1 03` | 1009 — exactly +240 | -Four properties fall out, each of which the "state" reading would have got wrong: +Four properties follow: - **The two bytes are one value.** Driving the low byte past `0xFF` carries into the high byte — `0x03F1` + 16 = `0x0401` — which independent bytes would not do. It also explains why a well-used file @@ -276,34 +286,32 @@ Four properties fall out, each of which the "state" reading would have got wrong - **The on-disk value lags the last write by one.** A statement's increment is not flushed until the *next* write, or until the connection closes (which lands the pending one plus its own). So a burst of *n* inserts reads as *n−1* until something follows it. Measure between two mid-burst samples and the lag cancels. -- **Every committed write costs exactly one**, DDL included. The lag makes this easy to misread: in the run - above `CREATE INDEX` appears to move it by 2 and the first `INSERT` by 0, but the whole seven-statement - sequence is 722 → 729, exactly +7. Isolating `CREATE INDEX` — the same trailing five inserts with and - without it — gives +5 versus +6, so it is one commit like anything else. +- **Every committed write costs exactly one**, DDL included. The lag makes this easy to misread: in the + sequence above `CREATE INDEX` appears to move it by 2 and the first `INSERT` by 0, but the whole + seven-statement sequence is 722 → 729, exactly +7. The same five inserts with and without a + `CREATE INDEX` differ by one. **Reopening does not reset it; compacting does.** The counter carries straight across a close and reopen (`…DA` before, `…DA` after). A DAO `CompactDatabase` writes a whole new file and its slot 1 starts at **256**, the idle value, then counts normally from there (744 → 256 → 260 after five inserts). -Tests: `CommitByteTableTests`. LibRed still does not read or maintain the table — ACE opens and queries -LibRed-created tables with the counter untouched — so this is documentation of the format, not a dependency. +LibRed does not read or maintain the table — ACE opens and queries LibRed-created tables with the counter +untouched. -This region is **undocumented by mdbtools and Jackcess** — LibRed's own decode, cross-checked three ways: -the white paper's Jet 2.x/3.x structure, the raw bytes of real ACE files, and the Microsoft **LDBView** -utility (Jet 2/3 only), which shows `1` for every unregistered slot — matching the idle `00 01`. +mdbtools does not document this region. The Microsoft **LDBView** utility (Jet 2/3 only) shows `1` for every +unregistered slot — matching the idle `00 01`. > **Creation must seed this.** A freshly created file has no users, so every slot must be the neutral > `00 01`, **not** zero — an all-zero table reads as "every user is mid-write," which Access rejects as > corrupt. `DatabaseCreator.BuildDefinitionPage` fills `0xE00`–`0xFFF` with the repeating `00 01`. -> LibRed itself does not read the table. ### 2.3 Creation date ⇄ security-SID coupling (verified) -Access opens the **workgroup file** (`System.mdw`, in `%AppData%\Microsoft\Access`) *before* the database — -confirmed with Process Monitor — and authenticates the current user against it. `System.mdw` is itself a Jet 4 -DB (identifier `"Jet System DB"`, version byte `0x01`) with **legacy Jet RC4 page encryption** (§2.4); LibRed -reads it directly. Its `MSysAccounts`/`MSysGroups` hold the **default-workgroup account SIDs**, which LibRed -now decodes to exactly the values Access shows (cross-checked against a VBA `Debug.Print` of the `SID` column): +Access opens the **workgroup file** (`System.mdw`, in `%AppData%\Microsoft\Access`) *before* the database and +authenticates the current user against it. `System.mdw` is itself a Jet 4 DB (identifier `"Jet System DB"`, +version byte `0x01`) with **legacy Jet RC4 page encryption** (§2.4); LibRed reads it directly. Its +`MSysAccounts`/`MSysGroups` hold the **default-workgroup account SIDs**, which LibRed decodes to exactly the +values Access shows in the `SID` column: | Account | kind | SID | |---|---|---| @@ -318,7 +326,7 @@ SID cluster opens cross-PC). Object ownership in a database uses the **"user" fo `Engine`/`Creator`. The 2-byte on-disk SIDs in `MSysACEs.SID` / `MSysObjects.Owner` are each a **workgroup account SID XOR'd with a -per-file 2-byte mask**. Verified against WideTable (mask `24-CC`): `Users 02-01 ^ 24-CC = 26-CD`, +per-file 2-byte mask**. Verified, e.g. with mask `24-CC`: `Users 02-01 ^ 24-CC = 26-CD`, `admin 03-01 ^ 24-CC = 27-CD` (read grantee), `Engine 03-03 ^ 24-CC = 27-CF` (system-object owner), `Creator 03-04 ^ 24-CC = 27-C8` (inheritable container grant). The long `Admins` SID isn't emitted — Access materialises it (as a 98-byte SID) on first open. @@ -326,22 +334,22 @@ materialises it (as a 98-byte SID) on first open. That mask is **bound to the exact millisecond-precise creation-date `double`** at `0x72`: a file with self-consistent SIDs but a *different* creation date is rejected with *"Record(s) cannot be read; no read permission on 'MSysObjects'/'MSysACEs'"* (Jet 3112). Grafting a real file's date **and** SIDs together opens -clean; either alone fails. There is **no closed-form `date → mask` function** — tested against all 144 -reference files (word XOR/sum, CRC-16, MSVCRT `rand`, VBA LCG, multiplicative hashes: 0 hits) and same-second -files have unrelated masks; Access most likely draws both the mask and the sub-second creation bits from one +clean; either alone fails. There is **no known closed-form `date → mask` function** — word XOR/sum, CRC-16, +MSVCRT `rand`, the VBA LCG and multiplicative hashes do not fit, and files created in the same second have +unrelated masks; Access most likely draws both the mask and the sub-second creation bits from one PRNG state, so they correlate but neither derives from the other. `DatabaseCreator` therefore **bakes one -verified `(SeedCreationDateBits, SidMask)` pair** (`0x40E68F1E8943D217` + `24-CC`, from WideTable) rather than -computing it — the from-scratch analogue of the account-SID constants. Limitations (deferred): every -LibRed-created file reports the same creation instant, and only the **default** workgroup is supported; -per-file-random dates and custom/secured workgroups both need the date↔mask coupling cracked (reading a -custom `System.mdw` itself now works — §2.4). +verified `(SeedCreationDateBits, SidMask)` pair** (`0x40E68F1E8943D217` + `24-CC`) rather than computing it — +the from-scratch analogue of the account-SID constants. Limitations (deferred): every LibRed-created file +reports the same creation instant, and only the **default** workgroup is supported; per-file-random dates and +custom/secured workgroups both need the date↔mask coupling cracked (reading a custom `System.mdw` itself +works — §2.4). ### 2.4 Legacy Jet 3/4 RC4 page encryption (verified) -The pre-ACE engine-level encryption (used by password-protected `.mdb` files and *always* by the `.mdw` +The pre-ACE engine-level encryption (used by *encoded* `.mdb` files and *always* by the `.mdw` workgroup file, which is why its account/password data isn't readable in a hex editor). The 4-byte **database key** at page-0 `0x3E` is the whole secret — there is **no password or key derivation** (unlike ACE Agile, -§2 above). Every page **except page 0** is RC4-encrypted with a per-page key of +§2.6 below). Every page **except page 0** is RC4-encrypted with a per-page key of ``` key = LE32(pageNumber XOR databaseKey) @@ -349,39 +357,35 @@ key = LE32(pageNumber XOR databaseKey) and the page bytes are the RC4 keystream XOR'd over the plaintext. This is the same per-page key mixing ACE Agile uses (`LE32(pageNumber) XOR encodingKey`), just feeding RC4 directly instead of deriving an AES IV. -Verified against a real `System.mdw` (`databaseKey = 0xABBB315C`): with XOR (not ADD) page-number mixing, -every page decrypts to a valid page-type byte (page 1 → `01` data, pages 2/3 → `02` TDEF, index pages → `04`), -`MSysObjects`/`MSysACEs` parse, and `MSysAccounts` yields the account SIDs in §2.3. Implemented as -`LibRed.Crypto.JetLegacyEncryption`; `PageChannel` selects it for non-ACE (`!IsAccdb`) files with a nonzero -database key. Regression tests in `JetLegacyEncryptionTests` (published RC4 vector + independent-oracle -key-derivation check). +Verified against a real `System.mdw`: the page-number mixing is XOR, not ADD; every page decrypts to a valid +page-type byte (page 1 → `01` data, pages 2/3 → `02` TDEF, index pages → `04`), `MSysObjects`/`MSysACEs` +parse, and `MSysAccounts` yields the account SIDs in §2.3. Implemented as `LibRed.Crypto.JetLegacyEncryption`; +`PageChannel` selects it for non-ACE (`!IsAccdb`) files with a nonzero database key. **Creating/removing the encoding (implemented).** `DatabaseEncryption.SetJetEncoding` picks a fresh random `0x3E` key, writes it (header-masked), and RC4s every page `1..n` in place; `RemoveJetEncoding` decrypts and clears the key (RC4 is symmetric, so this reuses the read codec). This is the "Encode Database" feature and is **completely independent of the database password** (§2, the `0x42` field): legacy Jet4 stores the encoding key -plainly at `0x3E` — it is *not* password-derived (verified: the encoded fixtures `2002encoded`/`db-enc` have a -nonzero `0x3E` key and an *empty* password; MSISAM/Money's password-derived key is a different format). A file may -carry both — encoding scrambles the pages, the password gates opening — and the `0x42` field lives on page 0 which -is never page-encrypted, so the two don't interact. Verified: encode-then-decode of `2002plain.mdb` is -byte-identical, the encoded output re-opens through the codec, and the password field survives encoding unchanged. -Jet 3 (`version byte 0x14 == 0`, 2048-byte pages) is rejected — unsupported. Tests: `LegacyJetPasswordTests`. - -**Combined encode + password — confirmed against a real Access file** (`2002encodedpw.mdb`, dbKey `0xaca0f84c`, -password `Test1`): LibRed opens Access's combined file and reads its table (our RC4 decode matches Access's encode); -the `0x42` password field is byte-identical to the password-only `2002plainpw.mdb`; and `RemoveJetEncoding` yields -valid re-openable plaintext. (`2002encodedpw` was made by copying `2002encoded` and adding a password, so it shares -that file's dbKey `0xaca0f84c` — this is *not* evidence of a derived key; the encoding key is treated as opaque per -file, read straight from `0x3E`.) Setting a password on an already-encoded file is identical to on a plain file, -since `0x42` is on the never-encrypted page 0. Access's "Encode Database" **also compacts** (`2002plain` 245760 → -`2002encoded` 237568; the later password add kept 237568), so a byte-for-byte reproduction from an un-compacted -source isn't achievable — our `SetJetEncoding` is a pure in-place RC4, a valid encoding without the compact. +plainly at `0x3E` — it is *not* password-derived (verified: an encoded file carries a nonzero `0x3E` key and an +*empty* password; MSISAM/Money's password-derived key is a different format). A file may carry both — encoding +scrambles the pages, the password gates opening — and the `0x42` field lives on page 0 which is never +page-encrypted, so the two don't interact. Verified: encode-then-decode is byte-identical, the encoded output +re-opens through the codec, and the password field survives encoding unchanged. Jet 3 (`version byte 0x14 == 0`, +2048-byte pages) is rejected — unsupported. + +**Combined encode + password — verified against a real Access file:** LibRed opens Access's combined file and +reads its table (the RC4 decode matches Access's encode); the `0x42` password field is byte-identical to the +same password on an unencoded file; and `RemoveJetEncoding` yields valid re-openable plaintext. The encoding key +is opaque per file, read straight from `0x3E`. Setting a password on an already-encoded file is identical to on +a plain file, since `0x42` is on the never-encrypted page 0. Access's "Encode Database" **also compacts** (adding +a password does not), so a byte-for-byte reproduction from an un-compacted source isn't achievable — LibRed's +`SetJetEncoding` is a pure in-place RC4, a valid encoding without the compact. ### 2.5 Office "Standard"/CryptoAPI page encryption (verified) -> **Why RC4 (background, per Wayne Phillips / EverythingAccess — explains the behaviour we verified).** Access 2007 -> encrypts at **page level but writes at record level** (record-level locking rewrites a single record, not the -> whole 4096-byte page), so it needs a **stream cipher** (encrypts byte-by-byte). Block ciphers (AES) can't do +> **Why RC4 (background).** Access 2007 encrypts at **page level but writes at record level** (record-level +> locking rewrites a single record, not the whole 4096-byte page), so it needs a **stream cipher** (encrypts +> byte-by-byte). Block ciphers (AES) can't do > partial-page writes, and RC4 is the only stream cipher in the standard Windows CSPs — hence Access 2007 is > RC4-only. **Access 2010's AES works only by forcibly disabling record-level locking**, and that path is Agile > (XML descriptor), *not* this binary AES-"Standard" descriptor. The default is Base provider / **RC4-40 / SHA-1**; @@ -392,13 +396,11 @@ source isn't achievable — our `SetJetEncoding` is a pure in-place RC4, a valid The pre-Agile `.accdb` encryption, carried by a **binary** `EncryptionInfo` header (version x.2, no XML) rather than the Agile XML — covering **RC4-CryptoAPI** and an **AES "non-standard"** variant. `LibRed.Crypto. OfficeStandardEncryption`; `PageChannel` selects it for an ACE file when no Agile descriptor is present. -Algorithm (matched to jackcess-encrypt and verified against real fixtures — db2007-oldenc = RC4-40 / `Test123`; -db-nonstandard = AES-256 / `password`): +Algorithm (verified against real RC4-40 and AES-256 files): The **hashing algorithm is parameterised by the header `AlgIDHash` field** (offset `h+12`), *not* fixed to SHA-1: -Access and third-party tools (e.g. EverythingAccess's "Encryption Manager for Access 2007") let the encryptor -pick it independently of the cipher. Verified values: `0x8003` MD5, `0x8004` SHA-1, `0x800c` SHA-256, `0x800d` -SHA-384, `0x800e` SHA-512 (MD2 `0x8001` / MD4 `0x8002` have no managed implementation → unsupported). The same +Access and third-party tools let the encryptor pick it independently of the cipher. Verified values: `0x8003` +MD5, `0x8004` SHA-1, `0x800c` SHA-256, `0x800d` SHA-384, `0x800e` SHA-512 (MD2 `0x8001` / MD4 `0x8002` have no managed implementation → unsupported). The same algorithm is used for `baseHash`, the per-block `H`, and the AES `0x36`/`0x5C` expansion. Let `hash = f(AlgIDHash)`: - `baseHash = hash(salt ‖ UTF16LE(password))`. @@ -421,25 +423,21 @@ algorithm is used for `baseHash`, the per-block `H`, and the AES `0x36`/`0x5C` e - cipher: RC4 (re-keyed per page; the verifier + verifier-hash decrypt as one continuous stream) or **AES-ECB**. The applicable `(key length, RC4 pad, AES iteration count)` is decided by whichever authenticates the verifier. -Fixture-free known-answer tests (real salt + verifier vectors, synthetic page 0) live in -`OfficeStandardEncryptionTests`; `DatabaseEncryptionTests` exercise generated RC4 key/hash variants end-to-end, -and `OfficeStandardVariantReadTests` mutate generated descriptors to verify clean rejection of unsupported -ciphers and hashes. The broader **RC4 and AES-128/192/256 × MD5/SHA-1/SHA-256/SHA-384/SHA-512 × -`KeySize=0`** sweep was verified against Access-tool re-encryptions of `db2007-oldenc` during format research. - -> **LibRed reads more than Access opens.** Verified on EverythingAccess-re-encrypted `db2007-oldenc` variants: -> **AES-128/AES-256 with MD5 or SHA-512 hashing** authenticate and decode correctly in LibRed, but **Access refuses -> them** ("your encryption settings are not valid — reinstall Microsoft Access"). Those files *are* validly +Verified across **RC4 and AES-128/192/256 × MD5/SHA-1/SHA-256/SHA-384/SHA-512 × `KeySize=0`**; unsupported +ciphers and hashes are rejected cleanly. + +> **LibRed reads more than Access opens.** Verified: **AES-128/AES-256 with MD5 or SHA-512 hashing** +> authenticate and decode correctly in LibRed, but **Access refuses them** ("your encryption settings are not valid — reinstall Microsoft Access"). Those files *are* validly > encrypted; Access's open path just whitelists cipher/hash combinations. Genuinely unsupported by both (LibRed > throws a clean `NotSupportedException`, never mis-reads ciphertext as plaintext): **3DES-168** (`0x6603`), > **3DES-112** (`0x6609`), **DES** (`0x6601`), **RC2** (`0x6602`), and **MD2 hashing** (`0x8001`, no managed impl). > These are weak legacy algorithms — deliberately not implemented. > -> **What a stock Access install opens (observed across all re-encrypted variants).** The `EncryptionEnhancer` -> add-in is required **only for the block ciphers** (AES/DES/3DES/RC2) — **RC4 always opens natively**: +> **What a stock Access install opens (observed).** The `EncryptionEnhancer` add-in is required **only for the +> block ciphers** (AES/DES/3DES/RC2) — **RC4 always opens natively**: > - **RC4, every key length (40–128) and every hash (MD2/MD4/MD5/SHA-1/SHA-256/384/512): opens with no add-in** -> (verified — the user opened all created RC4 files in Access). The descriptor may name the Enhanced RSA/AES -> *provider* (for >56-bit or SHA-2), but that CSP ships with Windows, so Access reads it fine. +> (verified). The descriptor may name the Enhanced RSA/AES *provider* (for >56-bit or SHA-2), but that CSP +> ships with Windows, so Access reads it fine. > - **AES/DES/3DES/RC2 (binary "Standard" descriptor): require the withdrawn EncryptionEnhancer COM add-in** to open > in Access. These are **validly encrypted** — a plain Access install refusing them (returning "incorrect > password" / "settings not valid") is a **deliberate guard**, not a validity failure: the add-in made Access @@ -454,10 +452,10 @@ ciphers and hashes. The broader **RC4 and AES-128/192/256 × MD5/SHA-1/SHA-256/S > record-level-locking corruption risk, so it can safely **read and write** these files. The only files LibRed does > **not** read that Access opens are > **RC4 + MD2/MD4** (`AlgIDHash 0x8001/0x8002`): .NET has no MD2/MD4, so those throw `NotSupported`. Implementing -> the two hashes from scratch (~130 lines) would close the gap; **deferred** as 1980s hashes unlikely in any real -> database. (Modern Access AES is Agile, not this binary AES-Standard descriptor — a separate codec.) +> the two hashes would close the gap; **deferred** as 1980s hashes unlikely in any real database. (Modern Access +> AES is Agile, not this binary AES-Standard descriptor — a separate codec.) -An encrypted file (`databaseKey != 0`) whose descriptor no codec recognises now fails with a clear +An encrypted file (`databaseKey != 0`) whose descriptor no codec recognises fails with a clear `NotSupportedException` in `PageChannel.Open` instead of decoding ciphertext as plaintext. Remaining unsupported: **Jet 3** (Access 97) encryption, which also needs Jet 3 format support (2048-byte pages). @@ -465,11 +463,11 @@ Remaining unsupported: **Jet 3** (Access 97) encryption, which also needs Jet 3 `EncryptionInfo` sits at a **fixed page-0 offset `0x29B`**, immediately preceded by a **2-byte blob length at `0x299`**. That length is **Access's "is this file encrypted?" signal**: on open Access reads `len@0x299` and, if nonzero, parses `len` bytes of `EncryptionInfo` at `0x29B`; if **zero it treats the file as unencrypted** — even -with a nonzero `0x3E` key and a valid descriptor present. Verified across `db-nonstandard`/`db2007-oldenc`/ -`db2013` (each length equals its exact blob size: 224 / 190 / 1055) and by experiment: a file with the key + -descriptor but `len@0x299 = 0` makes Access read ciphertext as plaintext and offer to "recover"; writing the -length makes it prompt for the password and open. LibRed likewise treats the length as authoritative: binary or -XML content outside the declared frame is ignored, and a frame extending beyond page 0 is rejected as malformed. +with a nonzero `0x3E` key and a valid descriptor present. Verified: the length equals the exact blob size, and a +file with the key + descriptor but `len@0x299 = 0` makes Access read ciphertext as plaintext and offer to +"recover"; writing the length makes it prompt for the password and open. LibRed likewise treats the length as +authoritative: binary or XML content outside the declared frame is ignored, and a frame extending beyond page 0 +is rejected as malformed. The Agile XML descriptor uses the same `len@0x299` + blob-at-`0x29B` framing. > **Creating encryption from scratch (implemented — Office Standard).** `LibRed.Crypto.DatabaseEncryption` @@ -480,7 +478,7 @@ The Agile XML descriptor uses the same `len@0x299` + blob-at-`0x29B` framing. > length signal, and encrypt every page. **Verified: AES-256, RC4-40, and Agile files created this way open in > the Access desktop GUI with the password.** For **Agile**, the same path emits the XML descriptor (version 4.4 > prefix `04 00 04 00 40 00 00 00` + UTF-8 XML) with a random data key wrapped via the 100000-spin SHA-512 KDF — -> Access's `.accdb` Agile has **no `` element** (verified against `db2013` and a created file), so +> Access's `.accdb` Agile has **no `` element** (verified), so > none is emitted. `ChangePassword` = decrypt + re-encrypt. **Jet 3 remains unimplemented.** Legacy Jet is > implemented, but under its own names rather than through `SetPassword`: an `.mdb` has two independent > mechanisms — the database password at `0x42` (`SetJetPassword`/`RemoveJetPassword`) and RC4 page encoding @@ -493,8 +491,8 @@ The Agile XML descriptor uses the same `len@0x299` + blob-at-`0x29B` framing. > name follows the CryptoAPI split: Base provider for RC4 ≤56-bit with MD5/SHA-1, else Enhanced RSA/AES. RC4 always > uses the truncated hash (no `0x36/0x5C` expansion) and only the 40-bit case zero-pads to 128. `keyBits == 40`, > `Sha1` reproduces the byte-identical Access-2007-default RC4-40 descriptor. **Every RC4 combo produced this way -> opens natively in Access** — no add-in needed (the add-in is only for the block ciphers). Round-tripped in -> `DatabaseEncryptionTests` across the key-length × hash matrix. +> opens natively in Access** — no add-in needed (the add-in is only for the block ciphers). Round-trips across +> the key-length × hash matrix. > **Writing to an existing encrypted database (implemented).** `IPageCodec.EncryptPage` is the inverse of > `DecryptPage`, so `PageChannel.WritePage` encrypts each page on the way to disk (page 0 stays clear) while the @@ -512,8 +510,8 @@ Access 2010+ "Set Database Password" on an `.accdb` uses **Office Agile encrypti the password (SHA-512 KDF, 100 000-spin), validates the verifier (wrong password → `UnauthorizedAccessException`), then decrypts each data page. **Access's one deviation from stock Agile:** the per-page IV block key is `LE32(pageNumber) XOR databaseKey` (the 4-byte key at `0x3E`), so `IV = SHA512(keyDataSalt ‖ blockKey)[:blockSize]` -and the page is `AES-256-CBC(dataKey, IV)`. Page 0 is never page-encrypted. Verified end-to-end against a -known-password fixture (decrypted pages match the unencrypted twin; `AgileEncryptionTests`). +and the page is `AES-256-CBC(dataKey, IV)`. Page 0 is never page-encrypted. Verified end-to-end: decrypted +pages match the unencrypted original. LibRed supports the exact Access profile verified from real files and its own Access-openable writer: AES-256-CBC, SHA-512, 16-byte salts/blocks, and `spinCount=100000`. These cleartext dimensions are validated diff --git a/src/LibRed/docs/format/page-01-data-and-rows.md b/src/LibRed/docs/format/page-01-data-and-rows.md index 18576994f..e9fa7c5f7 100644 --- a/src/LibRed/docs/format/page-01-data-and-rows.md +++ b/src/LibRed/docs/format/page-01-data-and-rows.md @@ -10,26 +10,44 @@ | `0x01` | 1 | Flags (observed constant `0x01`; the same byte appears on TDEF and index pages — verified) | | `0x02` | 2 | Free space | | `0x04` | 4 | Owning table's TDEF page — **or** the ASCII marker `LVAL` (`0x4C41564C`) for long-value pages | -| `0x08` | 4 | Jet4-only; purpose unknown — **zero** on every page observed (data, usage-map, LVAL). Jet3 has the row count here instead (which is why Jet4's row count sits 4 bytes later). LibRed writes zero. | -| `0x0C` | 2 | Row count on this page. **Capped at 256 in practice, whatever the free space** — see the slot-count limit below. | +| `0x08` | 4 | Jet4-only. **Zero on every page except the FIRST page of a multi-page long-value chain**, where it carries a stamp that must equal the pointing descriptor's own `0x08` — see [long-values](long-values.md). Jet3 has the row count here instead (which is why Jet4's row count sits 4 bytes later). LibRed writes it and checks it. | +| `0x0C` | 2 | Row count on this page. **ACE writes at most 255, whatever the free space** — see the slot-count limit below. | | `0x0E` | 2×N | Row slot directory: one 2-byte entry per row | -> **A data page holds at most 256 rows — a limit the free space does not express.** An index entry addresses a +> **A data page holds at most 255 rows — a limit the free space does not express.** An index entry addresses a > row as `page << 8 | row` ([page-03-04-index-btree.md](page-03-04-index-btree.md) §10.2), so the slot number > occupies exactly one byte and slots `0..255` are all that can ever be named. The field at `0x0C` is two bytes > wide and the page has room for far more, so nothing in the page format stops a writer going past it. > > Only narrow rows get there: 256 rows fit a 4 KB page once each is under about 14 bytes, which in practice -> means an all-fixed-column table of a few small columns. LibRed did exactly that and packed 314 rows onto a -> page — the rows wrote and scanned back correctly (a scan walks slots directly and never forms a pointer), but -> **every row past slot 255 was unaddressable by any index**, and its index entry aliased a different row on -> another page. A silent wrong answer on every indexed read, with no error anywhere. `RowInserter`'s page -> placer now refuses a page at `RowPointer.MaxRowsPerPage` instead of trusting free space alone; the guard is -> `IndexBuildOrderingTests`. +> means an all-fixed-column table of a few small columns. A writer that trusts free space alone will then +> overfill the page, and the rows still scan back through a slot-walking reader, since a scan never forms a +> pointer — which makes it look harmless. It is not: every row past slot 255 is **unaddressable by any +> index**, its entry aliasing a different row on another page, and **ACE cannot read those rows at all**. +> `RowInserter`'s page placer refuses a page at `RowPointer.MaxRowsPerPage` instead of trusting free space +> alone. > -> The pointer layout is ACE-verified, so the 256-slot ceiling follows from it. What is **not** measured is what -> ACE itself does on reaching it — whether it caps at 256 or lower, and whether it leaves the page in the -> free-pages map. LibRed's choice of 256 is the maximum the pointer allows, not an observation of ACE. +> > **ACE reads the full 16-bit count and then caps at 256 slots.** Pages of 400 / 400 / 100 rows read back +> > through ACE as **612** rows, exactly 256 + 256 + 100; a reader taking only the low byte would see +> > `0x90` = 144 and return 388, so the high byte is genuinely parsed — the rows beyond slot 255 are simply +> > dropped, with no error. Overfilling a page loses data to Access on a plain table scan, not merely on an +> > indexed read. +> +> **ACE's own ceiling is 255, one below what the pointer allows** (measured): ACE fills a page to exactly +> 255 rows and then starts another. It is not a space limit; a full page of single-`BYTE`-column rows still +> has **2,297 of its 4,096 bytes free**. ACE also **removes the page from the table's free-pages map** on +> reaching the cap, treating it exactly as it treats a full page — only a partial page stays in the map. So +> a reader must not infer remaining room from free space, and must not infer it from free-map membership +> either. +> +> **The row count really is two bytes, though nothing ACE writes proves it.** ACE never sets the high byte +> at `0x0D` — its own counts never exceed 255 — so ACE-written files alone cannot distinguish a two-byte +> field from a one-byte one. ACE nevertheless reads it as two bytes: pages holding **256** rows read back +> correctly (600 rows across three pages), where a one-byte reader would see `0x00` for those pages and +> report 88. +> +> LibRed's `RowPointer.MaxRowsPerPage` is therefore **255**, matching ACE rather than the pointer's maximum, +> so every page it writes is a shape Access also produces. Row slot entry: lower 13 bits (`& 0x1FFF`) = the row's byte offset in the page; `0x8000` = deleted, `0x4000` = overflow/lookup pointer (not an inline row). Rows are packed from the end @@ -42,7 +60,7 @@ the *later slots* — an ordering the delete path below depends on. ACE closes the gap rather than leaving the row in place: the rows below it slide up, their slot offsets follow, and the emptied slot becomes a **zero-length tombstone flagged deleted + overflow (`0xC000`) whose offset is the row's former end**. Slot *indices* never move, which is what keeps index entries and row ids -valid. Measured on three 19-byte rows, deleting each position in turn — free space rises by 19 every time: +valid. Three 19-byte rows, deleting each position in turn — free space rises by 19 every time: | deleted | directory after | | --- | --- | @@ -51,9 +69,9 @@ valid. Measured on three 19-byte rows, deleting each position in turn — free s | last | `0FED 0FDA CFDA` — nothing below to move | Pointing the tombstone at the former end rather than at the page end is what preserves the non-increasing -order, which is what makes it zero-length: its offset equals the preceding slot's. LibRed used to only set -the deleted flag and leave the row where it was, so the space was never reclaimed — about 21 bytes per -delete, permanently. `DeletedRowSpaceAccessTests`. +order, which is what makes it zero-length: its offset equals the preceding slot's. A writer that only sets +the deleted flag and leaves the row where it was never reclaims the space: every delete loses the row's +bytes for good. The **slot directory** is not reclaimed by either engine: a tombstoned slot is never reused, so a page that has seen thirteen rows carries thirteen slots whatever is live. Only the row bytes come back. @@ -63,7 +81,7 @@ forward pointer rather than the row, and the row itself sits on another page fla below). Deleting it reclaims the target first and then the pointer: ACE brings the target's page back to a bare `4080` free and returns the pointer's four bytes to the source page. Reclaiming only the pointer strands the moved row for ever — for a row that relocated because it grew, that is the whole widened row, -hundreds of bytes against the pointer's four. `RelocatedRowDeleteAccessTests`. +hundreds of bytes against the pointer's four. > **A record is capped at 4060 bytes**, counting everything in the row itself — the leading count, fixed > data, variable data, the offset table and the null bitmap — but not the payload of a Memo/OLE column, @@ -73,21 +91,20 @@ hundreds of bytes against the pointer's four. `RelocatedRowDeleteAccessTests`. > **The cap is ACE's, not the page's.** A page holds 4080 (4096 less the 14-byte header and a 2-byte slot), > and the 20-byte reserve below that is measured, not explained. It is also not derived from the row's > shape: three tables whose overhead differs by 23 bytes — 9, 12 and 20 text columns — all stop at the same -> 4060 (`RecordSizeAccessTests`). +> 4060. > > It is **the same 4060 under page-level and row-level locking** (`Jet OLEDB:Database Locking Mode` 0 and 1), > which is what it must be — a limit that moved with the connection would make a file written by one client > unreadable by another opening it differently. That independence is also what lets LibRed, which has no > locking mode at all, enforce one constant for every caller. > -> *Unverified lead, recorded because the evidence is suggestive rather than because it is established:* the -> 20 bytes may be space the engine always keeps for row-lock bookkeeping, whether or not the connection uses -> it. Two things point that way. Row-level locking arrived in Jet 4, the same generation as this reserve. -> And the **class** of error is wrong for corruption: a malformed row reads as "Unrecognized database -> format" or a decode failure, whereas this one reports a concurrent edit — so ACE parsed the row and then -> its multi-user layer objected, meaning something is *interpreting* those bytes rather than merely running -> past them. Nothing here tests it; the mode-independence above is equally consistent with the reserve being -> something else. +> *Unverified:* the 20 bytes may be space the engine always keeps for row-lock bookkeeping, whether or not +> the connection uses it. Two things point that way. Row-level locking arrived in Jet 4, the same generation +> as this reserve. And the **class** of error is wrong for corruption: a malformed row reads as +> "Unrecognized database format" or a decode failure, whereas this one reports a concurrent edit — so ACE +> parsed the row and then its multi-user layer objected, meaning something is *interpreting* those bytes +> rather than merely running past them. The mode-independence above is equally consistent with the reserve +> being something else. > > **Writing into the 4061–4080 band is worse than writing past it.** It fits the page, so nothing fails at > write time, and ACE then cannot materialise the row — it reports *"you and another user are attempting to @@ -109,18 +126,18 @@ A live slot with `0x4000` set **begins with** a 4-byte little-endian forward poi the same table. Its target slot has `0x8000` (deleted/hidden) set and `0x4000` clear: ordinary scans skip the hidden physical row, while the original row id and its index entries continue to resolve through the live source slot. A zero-length slot with both flags set is a tombstone, not a relocation -source. These shapes are verified by LibRed-created files opened by Access and Access-relocated files -read by LibRed. +source. These shapes are verified in both directions: Access opens LibRed's relocations and LibRed reads +Access's. **The slot is normally exactly 4 bytes wide, but not always.** When ACE relocates a row through -ordinary DML it trims the slot down to the pointer, and LibRed does the same. Measured across **317 -relocations with no exception**, covering ACE on x64, the **ACE 2010 runtime on x86**, and LibRed's -own writer, under: growing and shrinking text, repeated re-relocation of the same rows, page -fragmentation by interleaved deletes and re-inserts, and an OLE column going from NULL to a value. +ordinary DML it trims the slot down to the pointer, and LibRed does the same. Measured **with no +exception** on ACE x64, the **ACE 2010 runtime on x86**, and LibRed's own writer, under: growing and +shrinking text, repeated re-relocation of the same rows, page fragmentation by interleaved deletes and +re-inserts, and an OLE column going from NULL to a value. -Longer slots exist in the wild all the same. In the Northwind ACCDB, `MSysAccessStorage` carries live -overflow slots of 45–63 bytes. Their content is the row **as it was before it moved**, with only the -leading 4 bytes replaced by the pointer: +Longer slots exist in real files all the same — e.g. live overflow slots of 45–63 bytes in +`MSysAccessStorage`. Their content is the row **as it was before it moved**, with only the leading 4 bytes +replaced by the pointer: - discount those 4 bytes and every field lands exactly where the row format puts it — the pointer covers the 2-byte column count plus the first 2 bytes of the first column, leaving the remaining @@ -128,14 +145,13 @@ leading 4 bytes replaced by the pointer: - the remnant's `Id` / `ParentId` / `Type` / `Name` equal those of the row it forwards to; - the remnant's null bitmap differs from its target's in exactly one bit, the OLE column `Lv` — NULL in the remnant, set in the target — which is what grew the row and forced the move; -- every remnant is shorter than its target (55→89, 55→89, 63→87, 63→75, 45→57, 57→71, 53→65). +- every remnant is shorter than its target. So the slot kept the previous row's width and was stamped with the pointer rather than trimmed. -**What writes them is not known**, and is deliberately not asserted here. No write path reproduces -the shape — not the OLE-column transition the bytes themselves record, and not an older engine -(ACE 2010 trims exactly as the current one does). What has *not* been exercised is Access's own -maintenance of its system tables, which is not reachable through SQL DML. Readers must therefore -take the pointer from the leading 4 bytes and ignore any remainder rather than requiring a width. +**What writes them is not known.** SQL DML does not produce the shape on either ACE version — not even +the OLE-column transition the bytes themselves record; Access's own maintenance of its system tables, +which is not reachable through SQL DML, is unexamined. Readers must therefore take the pointer from the +leading 4 bytes and ignore any remainder rather than requiring a width. LibRed follows relocations through one shared resolver used by scans, index seeks, and raw-row mutation helpers. It validates that the source begins with a 4-byte pointer, plus the in-file page @@ -154,23 +170,45 @@ malformed pointers fail with `InvalidDataException`. ``` - **The variable section (offset table + `numVarCols` field) is OMITTED entirely when the table has no - variable columns.** An all-fixed row is just `[colCount][fixed][nullBitmap]` — verified vs ACE - (`AceModifyByteDiffProbe.Diff_libred_vs_ace_all_fixed_row_bytes`): `T(A,B,C LONG)` + row `(11,22,33)` is - **15 bytes** `03 00 | 0B000000 16000000 21000000 | 07`, not 19. The fixed-region length is recovered from the - schema (column offsets), so the row needs no var-data-start pointer. (A reader keyed on fixed offsets + + variable columns.** An all-fixed row is just `[colCount][fixed][nullBitmap]` — verified vs ACE: + `T(A,B,C LONG)` + row `(11,22,33)` is **15 bytes** `03 00 | 0B000000 16000000 21000000 | 07`, not 19. The + fixed-region length is recovered from the schema (column offsets), so the row needs no var-data-start pointer. (A reader keyed on fixed offsets + null bitmap decodes both forms; a *writer* must omit the section to be byte-faithful.) +- **An all-fixed row's fixed region is padded out to a minimum of 2 bytes**, so the shortest record ACE writes + is 5: `[colCount:2][fixed:2][nullBitmap:1]`. It is a **floor, not an alignment** — `T(A,B,C BYTE)` keeps its + odd 3-byte region (record 6) — and a row with a **variable trailer is exempt**: a `TEXT`-only table's region + stays 0 and a `BYTE`+`TEXT` one stays 1. Only regions of 0 or 1 are touched, and the pad is zero bytes + between the fixed values and the null bitmap. Booleans occupy no fixed bytes (below), so a table of nothing + but Booleans is the shape with a 0-byte region. Verified vs ACE: + + | table | natural | ACE | record | + |---|---|---|---| + | `A YESNO` | 3 | **5** | `01 00 \| 00 00 \| 01` | + | `A BYTE` | 4 | **5** | `01 00 \| 01 00 \| 01` | + | `A BYTE, B BYTE` | 5 | 5 | `02 00 \| 01 02 \| 03` | + | `A BYTE, B BYTE, C BYTE` | 6 | 6 | `03 00 \| 01 02 03 \| 07` | + | 9 × `YESNO` | 4 | **6** | `09 00 \| 00 00 \| FF 01` | + | `A TEXT(10)` | 13 | 13 | region 0, unpadded | + + The nine-Boolean row is what distinguishes the rule from a flat minimum *record* length: a 5-byte floor on + the record would leave it at 5, and ACE writes 6. **The TDEF keeps the true, unpadded fixed-row length** — + ACE stores 1 for a one-`BYTE` table while writing 5-byte rows into it — so this rounding happens at + row-write time and is not recoverable from the definition. + - Matching it is **not cosmetic**. Below 4 bytes **ACE misreads the record**: an all-Boolean table of ≤8 + columns encodes to 3 bytes without the pad, and ACE then reports every Boolean in it **False**, whichever + engine created the table — the fault is in the record, not the TDEF. A 4-byte record reads back correctly (16 Booleans, 2-byte bitmap), so the reader's cliff is one byte below what + ACE's writer guarantees. **Why** the floor is 2 rather than 1 is not established. + - **`colCount` is `max(column id) + 1`, not the live column count** — the two coincide only while ids are contiguous (a fresh table, or after ADD COLUMN, which keeps ids contiguous). They **diverge** once ids have a gap — a burned id from a type-change ALTER, or a DROP COLUMN gap — and then `colCount` (and therefore the null - bitmap width) is driven by the **highest id**, leaving bit positions for the dead ids. Verified vs ACE - (`AceModifyByteDiffProbe`): after `ALTER COLUMN B DOUBLE` burns B's id 1→3 in a 3-column table, the row's - `colCount` field is **4** and the null bitmap is `0x0F` (the dead id 1's bit is set present). A writer that + bitmap width) is driven by the **highest id**, leaving bit positions for the dead ids. Verified vs ACE: + after `ALTER COLUMN B DOUBLE` burns B's id 1→3 in a 3-column table, the row's `colCount` field is **4** and the null bitmap is `0x0F` (the dead id 1's bit is set present). A writer that sizes these by the live count writes a bit ACE can't find for any id ≥ live count → ACE reads that column null. - It is the **highest live id**, though, and *not* the TDEF's `0x29` id high-water — those differ when the highest-id column is dropped, and rows written afterwards then legitimately carry a *shorter* count and a - narrower bitmap than the rows before them. ACE reads across the change - (`VariableColumnHighWaterAccessTests`). + narrower bitmap than the rows before them. ACE reads across the change. - **Null bitmap** is indexed by **column id**; a **set bit = the value is present** (non-null). - **Fixed** column value is at `rowStart + 2 + fixedOffset`, `length` bytes. - A **fixed-length text** column (`CHAR`/`NCHAR`, not `TEXT`/`VARCHAR`) fills its whole `length`: the value is @@ -190,18 +228,16 @@ malformed pointers fail with `InvalidDataException`. index by variable index, so the row is written and read back happily by nothing at all. `0x2B` is the authority even where it exceeds `max(index) + 1` over the live columns: a row one slot short makes the `ALTER COLUMN` re-lay (§3.8) append the retyped column past the end of the row, and **ACE then rejects - the file** with *"A column Id is incorrect."* (`VariableColumnHighWaterAccessTests`.) + the file** with *"A column Id is incorrect."* - A variable **text**/**binary** value must **fit its column's declared width**. Where a fixed column pads or truncates, ACE **rejects** an over-long variable one — six characters into a `TEXT(5)`, six bytes into a `VARBINARY(5)`, both *"The field is too small to accept the amount of data you attempted to add"* (verified - vs ACE, `ColumnLengthAccessTests`). The bound is the descriptor's `length`, in **bytes** for both, so - `TEXT(5)` is 10. `Memo`/`OLE` are exempt — their inline form is a long-value descriptor whose size is + vs ACE). The bound is the descriptor's `length`, in **bytes** for both, so `TEXT(5)` is 10. `Memo`/`OLE` are exempt — their inline form is a long-value descriptor whose size is unrelated to `length`. - **Booleans** carry **no data** — the value *is* the null-bitmap bit (set = true). Boolean columns are never null, and they occupy **no fixed-region bytes**: their descriptor's fixed - offset is 0 and the fixed offsets of other columns skip over them. (Verified: Northwind's - `Products.Discontinued` is `fixed@0` even though it follows several fixed columns. A writer - must therefore *not* advance the fixed offset for a Boolean column.) On **write**, the value + offset is 0 and the fixed offsets of other columns skip over them. (Verified: a Boolean that + follows several fixed columns is still `fixed@0`. A writer must therefore *not* advance the fixed offset for a Boolean column.) On **write**, the value reaches the encoder as a bool *or* a number (a `bit` column is commonly inserted as `1`/`-1`/`0` or defaulted from `"0"`), so LibRed coerces it with Access truthiness — **any non-zero number (or bool true) sets the bit**, `0`/false clears it. Verified: Access reads LibRed-written bits back @@ -212,9 +248,8 @@ malformed pointers fail with `InvalidDataException`. > **Row-reader guardrails.** LibRed bounds the null bitmap and optional variable trailer before reading > them. The offset table must fit the row, entry 0 (the variable-data end) must not pass the table start, > and its end-first offsets must remain within the row and be non-increasing. LibRed permits unused bytes -> between entry 0 and the table itself; this preserves existing schema-evolved rows in the functional corpus, -> but the reason those rows retain the gap has not yet been verified against Access. A requested variable slot -> must exist. Fixed values must fit the derived fixed region, and fixed-width scalar codecs require the +> between entry 0 and the table itself, which schema-evolved rows can carry; why those rows retain the gap +> has not been verified against Access. A requested variable slot must exist. Fixed values must fit the derived fixed region, and fixed-width scalar codecs require the > exact widths in §6. A column id beyond an older row's stored `colCount` is absent/null—this preserves > ADD COLUMN behavior—and the variable trailer is considered present only when the stored row count > covers a currently known variable column. This also preserves the verified all-fixed form that omits diff --git a/src/LibRed/docs/format/page-02a-tdef.md b/src/LibRed/docs/format/page-02a-tdef.md index d7172f47b..0c027fd34 100644 --- a/src/LibRed/docs/format/page-02a-tdef.md +++ b/src/LibRed/docs/format/page-02a-tdef.md @@ -13,9 +13,9 @@ | `0x08` | 4 | TDEF length (total logical bytes) | | `0x0C` | 4 | Unknown — a constant `0x00000659` (1625) observed in every file | | `0x10` | 4 | Row count | -| `0x14` | 4 | **Highest AutoNumber value assigned** = the id of the last row inserted (the *next* id is this **`+ increment`**, see `0x18`); `0` when the table has no AutoNumber column. On a freshly created custom counter it is **`Seed - Increment`** so the first insert yields the `Seed` (verified: `COUNTER(1000, 7)` → `0x14` = `993`, first id `1000`). Verified **directly against `@@IDENTITY`**, and disambiguated from row count with a delete-gap: after inserting 3 rows, deleting id `3`, and inserting again (which is assigned id `4`, *not* reused `3`), `0x14` = `4` = the last inserted id while the row **count** is `3`. (Also: Northwind Categories = `8`, non-autonumber/text-PK tables = `0`.) mdbtools labels this *"Next autonumber value"* — that's **off by one**; the stored value is the last assigned, and the next id is `+ increment`. It is a **plain signed int32 that wraps** — there is no "counter exhausted" state (see the wrap note below). **Write requirement:** a writer inserting into an AutoNumber table must advance this to the last id it writes (LibRed does so in `RowInserter`); leaving it stale makes Access reissue an existing id and reject the insert as a duplicate primary key — verified end-to-end. | -| `0x18` | 4 | **AutoNumber increment** — a **signed 32-bit int** (same width as `0x14`); the step added to `0x14` for each new id. Default `1` (a plain `COUNTER`); a custom `COUNTER(seed, increment)` / `AUTOINCREMENT(seed, increment)` / `INTEGER IDENTITY(seed, increment)` sets it. **Confirmed a full int32, not a byte + 3 unknown** (verified vs ACE): `COUNTER(1, 300)` → `2C 01 00 00` (spans 2 bytes, ids `1, 301, 601`); `COUNTER(5, 100000)` → `A0 86 01 00` (3 bytes); and decisively `COUNTER(100, -5)` → `FB FF FF FF` = `-5` in two's-complement (all 4 bytes) with a **descending** sequence `100, 95, 90`. It reads `1` on every table (autonumber or not) because that is the default increment — mdbtools/Jackcess mislabel it a 1-byte constant / "autonumber enable" flag, which only *looks* right because the default increment is 1 (LibRed's own finding). The seed itself is not stored separately — it is recovered as `0x14 + increment` (correct on a freshly-created, un-inserted table). A writer/reader must treat it as a signed int32; the insert bump of `0x14` moves in the increment's direction (max for +, min for −) so a descending counter doesn't reissue an id — except at the int32 wrap, where the generated id is the correct continuation despite comparing as backwards (see the wrap note below). | -| `0x1C` | 4 | Complex-type AutoNumber (mdbtools `ct_autonum`) — the high-water value for a *complex* column (multi-value / attachment). `0` in every table observed; LibRed has no complex-column fixture to confirm a non-zero value (OLE DB DDL can't create such a column). **Read into `TableDef.ComplexAutoNumber` and written through `TdefBuilder` (0 for a table with no complex column) so it round-trips via the model, not only the raw surgery path** (`ComplexAutoNumberRoundTripTests`) | +| `0x14` | 4 | **Highest AutoNumber value assigned** = the id of the last row inserted (the *next* id is this **`+ increment`**, see `0x18`); `0` when the table has no AutoNumber column. On a freshly created custom counter it is **`Seed - Increment`** so the first insert yields the `Seed` (verified: `COUNTER(1000, 7)` → `0x14` = `993`, first id `1000`). Verified **directly against `@@IDENTITY`**. It is not the row count, and deleted ids are not reused: insert 3 rows, delete id `3`, insert again (assigned id `4`) → `0x14` = `4` while the row **count** is `3`. mdbtools labels this *"Next autonumber value"* — that's **off by one**; the stored value is the last assigned, and the next id is `+ increment`. It is a **plain signed int32 that wraps** — there is no "counter exhausted" state (see the wrap note below). **Write requirement:** a writer inserting into an AutoNumber table must advance this to the last id it writes (LibRed does so in `RowInserter`); leaving it stale makes Access reissue an existing id and reject the insert as a duplicate primary key — verified end-to-end. | +| `0x18` | 4 | **AutoNumber increment** — a **signed 32-bit int** (same width as `0x14`); the step added to `0x14` for each new id. Default `1` (a plain `COUNTER`); a custom `COUNTER(seed, increment)` / `AUTOINCREMENT(seed, increment)` / `INTEGER IDENTITY(seed, increment)` sets it. **Confirmed a full int32, not a byte + 3 unknown** (verified vs ACE): `COUNTER(1, 300)` → `2C 01 00 00` (spans 2 bytes, ids `1, 301, 601`); `COUNTER(5, 100000)` → `A0 86 01 00` (3 bytes); and decisively `COUNTER(100, -5)` → `FB FF FF FF` = `-5` in two's-complement (all 4 bytes) with a **descending** sequence `100, 95, 90`. It reads `1` on every table (autonumber or not) because that is the default increment — mdbtools labels it a 1-byte constant / "autonumber enable" flag, which only *looks* right because the default increment is 1. The seed itself is not stored separately — it is recovered as `0x14 + increment` (correct on a freshly-created, un-inserted table). A writer/reader must treat it as a signed int32; the insert bump of `0x14` moves in the increment's direction (max for +, min for −) so a descending counter doesn't reissue an id — except at the int32 wrap, where the generated id is the correct continuation despite comparing as backwards (see the wrap note below). | +| `0x1C` | 4 | Complex-type AutoNumber (mdbtools `ct_autonum`) — the high-water value for a *complex* column (multi-value / attachment). `0` in every table observed; a non-zero value is unverified (OLE DB DDL can't create such a column). **Read into `TableDef.ComplexAutoNumber` and written through `TdefBuilder` (0 for a table with no complex column) so it round-trips via the model, not only the raw surgery path** | | `0x20` | 8 | Unknown / reserved (zero observed) | | `0x28` | 1 | Table type: `0x4E` 'N' user, `0x53` 'S' system | | `0x29` | 2 | Maximum column count | @@ -40,22 +40,21 @@ > **`0x659` is a fixed constant, not a per-TDEF "definition id".** mdbtools labels the `0x0C` word > (and the column-descriptor `+0x01` / index-info `+0x00` markers) *"Matches definition block > unknown field"*, which could suggest a per-table id that these locations cross-reference. -> Verified otherwise: `0x0C` reads `1625` on **all 33 tables** of Northwind — user, system, -> complex-type, and hidden data tables — and on freshly ACE-created tables, and the header value -> equals the first column-descriptor marker in every one. So the "match" is simply that a shared -> constant appears in each spot, not a table-scoped identifier. (The mdbtools "*or 0*" variant was +> Verified otherwise: `0x0C` reads `1625` on user, system, complex-type and hidden data tables alike, +> including freshly ACE-created ones, and the header value equals the first column-descriptor marker in +> every one. So the "match" is simply that a shared constant appears in each spot, not a table-scoped +> identifier. (The mdbtools "*or 0*" variant was > not observed in any ACE table; it may be a Jet 3 or degenerate-record case.) -> ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (which hid the distinction during -> reverse-engineering) but differ for user tables. For **sizing the body**, which is this section's -> concern: `0x33` (real index count) sizes the index-data blocks **and** the `0x3F` pre-column block, while +> ⚠️ `0x2F` vs `0x33`: these are equal for MSysObjects (so it does not show the distinction) but +> differ for user tables. For **sizing the body**, which is this section's concern: `0x33` (real index count) sizes the index-data blocks **and** the `0x3F` pre-column block, while > `0x2F` (logical count) sizes the logical-index info blocks and the index names. Why the two differ, why > `0x33 ≤ 0x2F` always holds, and which of them the 32-index limit binds on are > [page-02d §3.5](page-02d-constraints.md). > **The AutoNumber counter wraps at the int32 boundary — there is no overflow error** (verified vs ACE -> OLE DB 16.0/12.0, `AceAutoNumberOverflowProbeTest`). `0x14` is an ordinary signed int32 and the next id is -> `0x14 + 0x18` computed **unchecked**, so an ascending counter runs +> OLE DB 16.0/12.0). `0x14` is an ordinary signed int32 and the next id is `0x14 + 0x18` computed +> **unchecked**, so an ascending counter runs > `… 2147483646, 2147483647, -2147483648, -2147483647 …` and a descending one mirrors it > (`-2147483648 → 2147483647`). ACE issues the wrapped id, writes it to `0x14`, and carries on — nothing in > the header records that the counter has been round the ring. This also happens without ever reaching the @@ -90,14 +89,15 @@ > **`ADD` / `DROP COLUMN` are metadata-only edits — the three column counts behave differently > (all probed vs ACE).** ACE never renumbers surviving columns or rewrites existing rows; a dropped > column's bytes become dead space, and an added column reads NULL on old rows (via the null bitmap). +> The one exception is an added **AutoNumber** column, which gives every existing row a value (below). > - **`0x2D` column count** — the **live** count. `DROP COLUMN` decrements it; `ADD COLUMN` increments it. > - **`0x29` maximum column count** — a **high-water** = the *next* column id to assign. `ADD COLUMN` > takes the current value as the new column's id, then increments `0x29`; `DROP COLUMN` **leaves it** > (dropped ids are **never reused**, so ids develop gaps, e.g. dropping id 1 leaves `0,2,3` and the next -> add is `4` — verified by dropping the highest column and observing the next id still continues past it). +> add is `4`; dropping the *highest* id does not free it either). > Because it never decrements, `0x29` is a hard **lifetime cap of 255**: once 255 ids have been handed out, -> `ADD COLUMN` fails even if the *live* count (`0x2D`) is lower — only a **compact** (which renumbers) frees -> the id space. ACE-verified: create 255 columns, drop 10, `ADD COLUMN` → *"Too many fields defined."* +> `ADD COLUMN` fails with *"Too many fields defined."* even if the *live* count (`0x2D`) is lower — only a +> **compact** (which renumbers) frees the id space. > LibRed enforces this on `0x29` (not the live count) rather than write a 256th id ACE can't represent. > **`ALTER COLUMN` consumes an id from `0x29` too**, keeping the column's ordinal position — so after a > modify, descriptor **position ≠ id**. That is a property of the ALTER mechanism rather than of this @@ -107,6 +107,27 @@ > increments it (the new column's variable index = the old value); `DROP COLUMN` of a variable column > **leaves it unchanged**, so survivors keep their stored variable index (§3.4) and existing rows keep the > same number of variable slots. (A fixed column doesn't touch `0x2B`.) +> - **The long-value list (§3.3.2)** — `DROP COLUMN` of a memo/OLE column removes its 10-byte entry; the other +> entries keep their `col_num` and map pointers. Its maps and pages are retired as described in +> [long-values](long-values.md#dropping-a-long-value-column). +> - **Past the new end** — whenever ACE rewrites a single-page definition, shrinking or growing, it zeroes +> exactly **8 bytes** past the new definition length (`0x08`) — the trailing reserve — and leaves every byte +> beyond them as it was, so an old definition's tail stays on the page. Measured on single-page definitions +> for `DROP COLUMN` of a fixed, a text, a memo and an OLE column and for `DROP INDEX`, each with and without +> long-value columns; on a two-page definition falling back to one; and, growing in place over such a stale +> tail, for `ADD COLUMN` and for a relationship's incoming block (the reserve's stale bytes zeroed). A definition that stays multi-page moves its continuation data to fresh pages instead (§3.2, +> "Rewriting a multi-page TDEF"). Nothing reads these bytes; LibRed leaves them as ACE does. (A new table's +> definition, from `CREATE TABLE`, is written onto a zeroed page — ACE's handling of a reused page there is +> not measured.) +> +> **An AutoNumber added to a table that already has rows numbers them** (verified vs ACE). The existing rows +> take `1, 2, 3 …` in table order, whatever the column's seed and increment. `0x14` is then set as follows: +> - **Default seed 1, increment 1** — `COUNTER`, `COUNTER(1, 1)`, `AUTOINCREMENT`, `IDENTITY`, however +> spelled: `0x14` = the number of rows, so the next insert continues after them (two rows take `1` and `2`, +> the next insert `3`). +> - **Any other seed or increment**: `0x14` = `Seed - Increment`, as on a new table, so the next insert +> starts at the seed even where that repeats a value the rows were given — `COUNTER(2, 1)` over two rows +> goes on `2, 3, 4`, and `COUNTER(1, 5)` goes on `1, 6, 11`. > > An added **fixed** column's fixed offset is the current end of the fixed region (`max(offset+length)`); > an added **variable** column appends. A dropped column's descriptor + name are removed from the column @@ -115,10 +136,9 @@ #### Which writers must honour each of these -Every rule above is a property of the **format**, so it binds every path that writes, not just the one it -was first measured on. That is not obvious from the prose: the rules are stated once, here, while the code -that must obey them is spread across three row writers and five definition mutators — and each new path -tends to re-derive the rule from the *live* columns, which is the one reading that is always wrong. +Every rule above is a property of the **format**, so it binds every path that writes. The rules are stated +once, here, while the code that must obey them is spread across the row writers and definition mutators — +and a path that re-derives a rule from the *live* columns gets it wrong. The table is the enforcement surface. A blank cell means the path cannot reach that invariant, not that it is exempt. @@ -132,23 +152,20 @@ is exempt. | fixed region never shrinks below existing rows | ✅ `InferFixedDataLength` takes `max(pinned, derived)` | ✅ derived from the old row | | | | | column id from the `0x29` high-water, 255 lifetime cap | | | ✅ | ✅ leaves `0x29` | ✅ burns an id | -**Audited, and two of the cells were wrong when the table was first drawn up** — the two row-writer cells -for the variable-slot count. `RowEncoder` packed the chunks densely and `BuildRelaidRecord` appended onto -the short row that produced, so a `DROP COLUMN` of a variable column silently moved every later column -down one slot, and dropping the *last* variable column then retyping another made ACE reject the file -outright. Both are fixed; the row is `VariableColumnHighWaterAccessTests`. +**The variable-slot count is the easiest cell to get wrong on the row writers.** Packing the variable +chunks densely (one per live variable column) means a `DROP COLUMN` of a variable column silently moves +every later column down one slot, and dropping the *last* variable column then retyping another makes ACE +reject the file outright. -The remaining cells were checked the same way rather than by reading — ACE performing the identical DDL, -compared field by field, and ACE reading rows written on both sides of the drop. Two are worth recording -because the obvious guess is wrong: +Every cell is verified against ACE performing the identical DDL and reading rows written on both sides of +the drop. Two more are worth recording because the obvious guess is wrong: - **A row's `colCount` is `max(live id) + 1`, not this page's `0x29` high-water** — they differ once the highest-id column is dropped. The row-side consequence is [page-01 §5](page-01-data-and-rows.md). - **The fixed half follows the OPPOSITE rule to the variable half.** A dropped variable column's index is abandoned and the next added column goes *above* it; a dropped fixed column's offset is **reused** by the next fixed column added. `F(K, P, Q LONG, T TEXT)`, drop `Q`, add `R LONG` → ACE puts `R` at offset 8, - where `Q` was. Nothing about one half predicts the other, which is exactly why deriving the variable - section from the live columns looked reasonable. + where `Q` was. Nothing about one half predicts the other. ### 3.2 Multi-page TDEFs @@ -158,9 +175,24 @@ Reassemble before parsing: take the **first page whole**, then append each conti page's bytes **from offset 8** (continuation pages have an 8-byte header). Column offsets are absolute from the first page, so parsing is otherwise unchanged. +**The chain holds the definition and then its 8-byte trailing reserve** (verified vs ACE). Every page is +filled before the next begins, and the reserve follows the last definition byte — so when the definition ends +within 8 bytes of a page's end, the reserve spills onto a further page, which then holds **reserve bytes and +no definition**. The page count is therefore set by `length + 8`, not by the length: + +| definition length | pages | continuation free space | +| --- | --- | --- | +| 4,088 | 1 | — (page 1 exactly full: 4,088 + 8) | +| 4,090 | 2 | 4,086 (two reserve bytes) | +| 4,096 | 2 | 4,080 (the whole reserve, no definition) | +| 4,098 | 2 | 4,078 (two definition bytes + the reserve) | +| 8,181 | 3 | page 2: 0 (4,085 definition bytes + 3 of the reserve); page 3: 4,083 (the other 5) | + +Each page's free space is what it has left once its definition and reserve bytes are placed. + LibRed uses one shared reader for catalog parsing, index-root updates, and DDL surgery. It treats the -`0x08` definition length as authoritative: the length determines the exact number of continuation -pages and the exact number of bytes copied from the final page. Every page number must be in-file, +`0x08` definition length as authoritative: `length + 8` determines the exact number of continuation +pages, and the length the exact number of bytes copied from the final page. Every page number must be in-file, the chain must be acyclic, continuation headers must be `[02 01]`, and the chain must be neither shorter nor longer than the declared length. LibRed additionally applies a **1 MiB per-definition safety budget** before allocation. That budget is an implementation hardening limit—not a newly @@ -170,12 +202,23 @@ the documented 255-column, 32-index, and 64-character-name limits. > **Writing a multi-page TDEF (verified vs ACE).** The 8-byte continuation header is > `[0x02][0x01][free space: 2][next page: 4]` (page type, flags, then the same `0x02` free-space and > `0x04` next-page fields as page 1). The **first page is filled completely** (free space `0`) and its -> `0x04` points to the first continuation; each continuation carries `PageSize − 8` bytes of definition -> data (from offset 8), the **last** one leaving the usual 8-byte trailing reserve — so its free space is -> `PageSize − 8 − dataLen − 8`. The definition-length field (`0x08`, on the first page) is the **total** -> length across all pages. LibRed writes this in `TableCreator.WriteDefinition`, used when `CREATE INDEX` -> grows a definition past one page (confirmed: a 30-column, 30-index table spills to one continuation -> page, `defLen 4115`, exactly as ACE writes it, and Access reads all 30 indexes). +> `0x04` points to the first continuation; each continuation carries up to `PageSize − 8` bytes (from offset +> 8), definition first and then the reserve, as above — so a page the reserve fits on after its last data has +> free space `PageSize − 8 − dataLen − 8`, and a page that is full has `0`. The definition-length field +> (`0x08`, on the first page) is the **total** length across all pages. LibRed writes this in `TableCreator.WriteDefinition`, used when `CREATE INDEX` +> grows a definition past one page (verified: a 30-column, 30-index table spills to one continuation +> page, `defLen 4115`, exactly as ACE writes it, and Access reads every index). + +> **Rewriting a multi-page TDEF (verified vs ACE: `ADD COLUMN`, `DROP COLUMN`, `CREATE INDEX`).** Growing or +> shrinking, ACE rewrites the **first page in place** but writes the continuation data to **newly allocated +> pages**, and releases the old continuation pages without touching a byte of them (they keep type `0x02`); +> they are back in the global free-pages map once the session closes. The new last page is zero past its +> data. The **last continuation is allocated first**, each allocation taking the lowest free page: from free +> pages `354…` a two-continuation chain became `first → 355 → 354`, and from the end of a file +> `first → n+1 → n` (a `CREATE TABLE`'s chain runs the same way). When a definition falls back to one page, +> the first page's `0x04` next pointer is zeroed and it follows the single-page rule below (8 bytes zeroed +> past the new end, the rest left); a definition growing onto a second page keeps its first page and takes a +> fresh continuation. LibRed's `WriteDefinition` does all of this. ### 3.3 Body layout (in order, after the header) @@ -190,7 +233,7 @@ the documented 255-column, 32-index, and 64-character-name limits. column usage maps (per long-value column) × 10 bytes, then 0xFFFF (§3.3.2) ``` -> **Object-name limits (verified vs ACE OLE DB 2026-07-12).** The 2-byte length prefix could physically hold a +> **Object-name limits (verified vs ACE OLE DB).** The 2-byte length prefix could physically hold a > 65535-byte name, but ACE enforces **64 characters** for table/column/index names — a longer name makes ACE > reject the *entire file* (65+ char column → "Unrecognized database format"; 65+ char table → "Unspecified > error"), not just the object. ACE's *storage/read* path tolerates every special character (quotes, `#`, `%`, @@ -204,8 +247,8 @@ the documented 255-column, 32-index, and 64-character-name limits. ### 3.7 Writing a TDEF Access accepts (verified) Every field documented above is part of the format and must be written — **including the constants -and markers the reader ignores** (`0x01` flags, the `0x0659`/`0x0783` markers, the en-US locale -`0x0409`, the `0x80`/`0x08` index-flag bits, …). The reader being lenient about a field does **not** +and markers the reader ignores** (`0x01` flags, the `0x0659`/`0x0783` markers, each column's collation +bytes, the `0x80`/`0x08` index-flag bits, …). The reader being lenient about a field does **not** make it optional on write; Access validates them when it opens the table. With every documented field populated, a LibRed-written TDEF matches an ACE-created one **byte-for-byte** (verified by diffing; only page numbers and the auto-generated index name differ). @@ -227,10 +270,9 @@ Only a few fields are *not* fixed constants and so warrant a write note: index's map covers **every page of that index's B-tree** — root, internal nodes and leaves — not just the root. The root's bit is set at **CREATE**, before any row exists (verified: a freshly created empty index has exactly its root bit set); thereafter every page a split allocates is added, so the - union of a table's index maps equals exactly the set of index pages present (verified against ACE: - union == owned index pages, byte-for-byte, incl. a 4000-row load that splits both trees several - levels). LibRed reproduces this: `IndexWriter.AllocateIndexPage` marks each page it allocates during a - split, and `TableCreator` marks the root at creation (both `CreateTable` and `CREATE INDEX`). + union of a table's index maps equals exactly the set of index pages present (verified byte-for-byte + against ACE, including loads that split the trees several levels). LibRed reproduces this: + `IndexWriter.AllocateIndexPage` marks each page it allocates during a split, and `TableCreator` marks the root at creation (both `CreateTable` and `CREATE INDEX`). Note this map is **advisory for LibRed's own reads** — `IndexWriter` navigates the B-tree structurally (root child-pointers + leaf next-pointers), never by the map — but Access's maintenance relies on it, and it feeds the owned-map page-budget calculation (a growing index map shrinks the owned map's room; @@ -241,16 +283,16 @@ Only a few fields are *not* fixed constants and so warrant a write note: case the map goes on a page of its own, as ACE's does (see the multi-page distribution rule in [long-values.md](long-values.md)) — then **back-fills** the B-tree by scanning every existing row (`AddEntry` per row). Verified vs ACE: a primary key added after data - enforces uniqueness and seeks correctly, incl. a 2000-row back-fill that splits the tree. + enforces uniqueness and seeks correctly, including a back-fill that splits the tree. -> **Access now opens and round-trips a LibRed-created table** (empty `COUNT`, `INSERT`, read-back — -> verified through the ACE OLE DB provider). Getting there required *all* of the following together; -> each was independently necessary (removing any one reproduces "Unrecognized database format"): +> **Access opens and round-trips a LibRed-created table** (empty `COUNT`, `INSERT`, read-back — +> verified through the ACE OLE DB provider) only when *all* of the following hold together; each is +> independently necessary (omitting any one gives "Unrecognized database format"): > > 1. **TDEF byte-validity** — every constant/marker written (§3.1), and the trailing `0xFFFF` that > terminates the **long-value usage-map list** (§3.3.2 — *not* the index names, which precede it) > included in the definition length. It is mandatory even on a table with no long-value columns at all; -> [long-values.md](long-values.md) owns the rule and the byte-diff that found it. +> [long-values.md](long-values.md) owns the rule. > 2. **Global page allocation** — pages must be taken from the database's **global free-pages map** > (§9.1), not by blindly growing the file, so Access accounts for them. LibRed allocates by > clearing a free bit there. diff --git a/src/LibRed/docs/format/page-02b-columns.md b/src/LibRed/docs/format/page-02b-columns.md index 4e0fa1226..f48285204 100644 --- a/src/LibRed/docs/format/page-02b-columns.md +++ b/src/LibRed/docs/format/page-02b-columns.md @@ -10,11 +10,12 @@ | `0x01` | 2 | Record marker `0x0659` (see §3.1 note); ignored | | `0x03` | 2 | Unknown (zero observed) | | `0x05` | 2 | Column id | -| `0x07` | 2 | Variable-table index. For a **fixed** column it is the running count of variable columns with a smaller id (**not** `0`) — measured on ACE's own `ADD COLUMN`, which writes `2` for a LONG added to `(K LONG, A TEXT, B TEXT)`. For a **variable** column it is that column's own slot index, which is the `0x2B` **high-water** and *not* the count of live variable columns: after a variable column is dropped the next one goes above the abandoned slot, so the two part company (`VariableColumnHighWaterAccessTests`). Verified byte-for-byte against `MSysObjects`/`MSysACEs` in a real DAO file, and against ACE performing the same DDL. | +| `0x07` | 2 | Variable-table index. For a **fixed** column it is the running count of variable columns with a smaller id (**not** `0`) — ACE's own `ADD COLUMN` writes `2` for a LONG added to `(K LONG, A TEXT, B TEXT)`. For a **variable** column it is that column's own slot index, which is the `0x2B` **high-water** and *not* the count of live variable columns: after a variable column is dropped the next one goes above the abandoned slot, so the two part company. Verified byte-for-byte against DAO-written system tables and against ACE performing the same DDL. | | `0x09` | 2 | Column number — a second copy of the column id `0x05` on a **user** table, but **zero** on the tables the engine writes for itself (see the note below). It **diverges after an `ALTER COLUMN` type change**, which burns a new id into `0x05` yet leaves `0x09` at the *old* id; see §3.8 | -| `0x0B` | 1 | Numeric **precision** (Decimal/Numeric columns); otherwise the low byte of the locale id, `0x09` | -| `0x0C` | 1 | Numeric **scale** (Decimal/Numeric columns); otherwise the high byte of the locale id, `0x04` | -| `0x0D` | 2 | Text sort-order **version** — a 2-byte field (the high half of a 4-byte sort-order descriptor whose low half is the locale at `0x0B`, `0x0409` = General, §10.4). The version *number* is the **high byte at `0x0E`**: `0` = General Legacy (Access 2000–2007), `1` = the "General" order Access 2010+ made default (a different key encoding). The **low byte `0x0D` is `0` in every file observed** and isn't modelled — but the field is nominally 2 bytes, so keep an eye on it (see note). | +| `0x0B` | 1 | Numeric **precision** (Decimal/Numeric columns); otherwise the low byte of the collation's LANGID (the database default, e.g. `0x09` for en-US) | +| `0x0C` | 1 | Numeric **scale** (Decimal/Numeric columns); otherwise the high byte of the LANGID (`0x04` for en-US) | +| `0x0D` | 1 | Collation **sort id** — the LCID's high word; `0` except for an alternate sort order (see the note below) | +| `0x0E` | 1 | Collation **sort-order version**: `0` = General Legacy (Access 2000–2007), `1` = the "General" order Access 2010+ made default (a different key encoding, §10.4) | | `0x0F` | 1 | Flags (see below) | | `0x10` | 1 | Extended flags: `0x01` compressed-Unicode capable, `0xC0` calculated column | | `0x11` | 4 | Unknown (zero observed) | @@ -24,28 +25,25 @@ **Flags (`0x0F`):** `0x01` fixed-length, `0x02` updatable, `0x04` auto-number, `0x40` auto-number GUID, `0x80` hyperlink (on a Memo column). -> **`0x09` is written by everything that creates a user table, and only by those.** Measured per table -> across the fixtures: every genuine user table carries the id on every column (Northwind's `Categories`, -> `Customers`, `Employees`, `Orders`, …; `Ace16Types`' `T`), and every zero belongs to a table the engine -> made for itself — `MSysObjects`, `MSysACEs`, `MSysQueries`, `MSysRelationships`, `MSysComplexColumns`, -> `MSysComplexType_*`, and the `f__Data` complex-column backing tables. `Database4.accdb` looks like -> a counterexample at a glance because it is *all* zeros; it simply contains no user tables at all. -> (`MSysAccessStorage` goes each way depending on the file, so it is not part of the bootstrap set.) +> **`0x09` is written by everything that creates a user table, and only by those.** Every genuine user +> table carries the id on every column, and every zero belongs to a table the engine made for itself — +> `MSysObjects`, `MSysACEs`, `MSysQueries`, `MSysRelationships`, `MSysComplexColumns`, +> `MSysComplexType_*`, and the `f__Data` complex-column backing tables. A database with no user +> tables is therefore *all* zeros. (`MSysAccessStorage` goes each way depending on the file, so it is not +> part of the bootstrap set.) > -> Three creators were tried and all three write the id: ACE's SQL DDL, DAO's object model -> (`CreateTableDef`/`CreateField`/`Append`, the path Access's UI uses) and DAO-executed SQL. **Compacting a -> database preserves the field exactly** — before and after are byte-identical — so it is fixed at creation -> and no later rewrite normalises it. LibRed wrote zero everywhere until this was measured, on the strength -> of a comment that had generalised from the system tables; it now writes the id except on a system column. -> `ColumnDescriptorByteParityAccessTests`. +> ACE's SQL DDL, DAO's object model (`CreateTableDef`/`CreateField`/`Append`, the path Access's UI uses) +> and DAO-executed SQL all write the id. **Compacting a database preserves the field exactly** — before and +> after are byte-identical — so it is fixed at creation and no later rewrite normalises it. A writer that +> generalises from the system tables and writes zero everywhere is wrong; LibRed writes the id except on a +> system column. > **Date/Time Extended carries only the primary language id.** For a `DATETIME2` column ACE writes the > **low byte** of the database's LANGID at `0x0B`/`0x0C` — the primary language with the sublanguage half > cleared — and zero at `0x0D`/`0x0E`, where every other type carries the whole LANGID. The value is 42 > bytes of ASCII (§6), so there is nothing to collate. > -> Measured across five collating orders, each produced by compacting a database into it (DAO's -> `CompactDatabase`, the documented way to change one), with a `TEXT` column in the same table as control: +> Verified across collating orders, with a `TEXT` column in the same table for comparison: > > | database LANGID | `TEXT` `0x0B`/`0x0C` | `DATETIME2` `0x0B`/`0x0C` | > | --- | --- | --- | @@ -55,24 +53,22 @@ > | `0x040E` Hungarian | `0E 04` | `0E 00` | > | `0x041D` Swedish | `1D 04` | `1D 00` | > -> An en-US database alone reads as the constant `0x0009`, which is how this was first mis-implemented; the -> en-GB row is the tell, a different LANGID giving the same value because it shares en-US's primary id. +> On an en-US database alone this reads as the constant `0x0009`, but it is not a constant: en-GB gives the +> same value only because it shares en-US's primary id. > Every sort order Access offers has a primary id below `0xFF`, so "low byte" and Windows' `PRIMARYLANGID` -> (mask `0x3FF`) cannot be told apart here. `DateTime2LocaleAccessTests`. +> (mask `0x3FF`) cannot be told apart here. > **Every documented flag is modelled — nothing rides through raw except the reserved/unknown.** LibRed reads > each `0x0F` bit and the whole `0x10` byte into `ColumnDef` (`IsUpdatable`/`IsGuidAutoNumber`/`IsHyperlink`, > `SupportsCompressedUnicode`/`IsCalculated`) and composes them back on write, so they round-trip explicitly. > The only bytes preserved verbatim through `ColumnDef.RawDescriptor` are the genuinely reserved/unknown ones: > the reserved words at `0x03` and `0x11`, and any *undocumented* bits of `0x0F`/`0x10` (zero in every file -> observed). `ColumnDescriptorFlagTests`. +> observed). > **Nullability, defaults and checks are *not* in the descriptor.** The column's *Required* (NOT NULL) -> property is **not** encoded anywhere in the 25-byte descriptor — verified against Northwind: a nullable -> column (`Orders.ShippedDate`) and a non-null column of the same type (`Orders.OrderDate`) have -> **byte-identical** descriptors; the flag byte `0x0F` only ever distinguishes fixed-length -> (`0x01`), the always-set updatable bit (`0x02`), and auto-number (`0x04`), while the extended -> flags (`0x10`) and reserved bytes (`0x03`, `0x0D`) are zero for every column. `Required`, +> property is **not** encoded anywhere in the 25-byte descriptor — verified: a nullable column and a +> non-null column of the same type have **byte-identical** descriptors, and no flag in `0x0F` or `0x10` +> means nullability. `Required`, > `DefaultValue`, and `CheckConstraints` instead live in the table's **column-properties blob** (Jet's > per-object extended properties, a.k.a. `LvProp`, stored in the `MSysObjects` row). LibRed reads and > writes all three, byte-for-byte vs ACE — the **on-disk `LvProp` format** is documented in @@ -138,7 +134,7 @@ as 20 bytes of raw UTF-16LE. > bitmap bit, but a calculated one carries a real payload (`FF`/`00`) and its bit is set even when the > value is False. > -> **The declared length at `0x17` is a constant per result-type family, not a payload size.** Swept across +> **The declared length at `0x17` is a constant per result-type family, not a payload size.** Across > every type DAO will create: `0` for a Memo result, `509` for Text at *any* requested size (10, 60 and 255 > all give 509), `510` for Binary, and `39` for everything else including GUID — which, like Memo and Binary, > takes descriptor type `0x0A` Text, so only the length tells the three apart. Nothing moves these: not the @@ -148,27 +144,25 @@ as 20 bytes of raw UTF-16LE. > **A calculated Memo arrives through the long-value machinery.** ACE declares it `0x0A` Text with a > declared length of `0`, gives it a **long-value map entry**, and stores a long-value descriptor in the > slot; the envelope above is what that descriptor resolves to. So a long-value map may name a column whose -> declared type is not Memo/OLE — a guard that assumed otherwise rejected the TDEF, and because the catalog -> loads every TDEF that made the whole database unopenable. +> declared type is not Memo/OLE — a reader that rejects such a TDEF makes the whole database unopenable, +> because the catalog loads every TDEF. > > That descriptor obeys the ordinary long-value rules: the value inlines while it is at most 64 bytes and > otherwise takes its own LVAL page. What is measured against that limit is the **whole envelope**, 23 bytes > of frame plus the uncompressed payload, so a calculated Memo crosses it sooner than the text alone > suggests — `"hello-memo"` inlines at 43 bytes, while a 36-character result is 95 and spills. > -> Measured against ACE-authored columns of every type DAO will create (`CalculatedColumnAccessTests`), and -> against `AdventureWorks_Learn_To_Write_DAX.accdb`, a database in the wild whose `fctSales.OrderDate` is -> a calculated `DateTime`. LibRed reads **and writes** these values: it evaluates the expression itself and -> refreshes the cache on exactly the writes ACE would — when an UPDATE touches a column the expression reads, -> and not otherwise. It also **creates** them, and ACE accepts, evaluates and recomputes the result for every -> result type (`Creates_a_calculated_column_ace_accepts`). Three things must agree or Access reads the payload -> at the wrong width: the descriptor's promoted type, the constant declared length, and `ResultType` in the +> Verified against ACE-authored columns of every type DAO will create. LibRed reads **and writes** these +> values: it evaluates the expression itself and refreshes the cache on exactly the writes ACE would — when +> an UPDATE touches a column the expression reads, and not otherwise. It also **creates** them, and ACE +> accepts, evaluates and recomputes the result for every result type. Three things must agree or Access +> reads the payload at the wrong width: the descriptor's promoted type, the constant declared length, and `ResultType` in the > blob. A Memo result additionally needs long-value maps, which its *declared* type does not indicate. > `0x0B`–`0x0C` is a union keyed by type: for a Decimal/Numeric column (type `0x10`) it holds > the **precision** (`0x0B`) and **scale** (`0x0C`) — verified with a `DECIMAL(12,3)` column, -> which reads precision = 12, scale = 3; for every other type it reads the constant `0x0409` -> (the en-US LCID / text collation). +> which reads precision = 12, scale = 3; for every other type it holds the collation's LANGID (below) — +> the database default, so `0x0409` only on an en-US database. > > **`0x0B`–`0x0E` is a 32-bit Windows LCID with the sort-order version in its unused top byte.** For > non-numeric columns the four bytes are: @@ -185,30 +179,23 @@ as 20 bytes of raw UTF-16LE. > **The sort id is what separates an alternate sort order from its base locale.** `German Phone Book` is > `0x00010407` against German's `0x00000407`; `Hungarian Technical` is `0x0001040E` against Hungarian's > `0x0000040E`; `Georgian Modern` is `0x00010437`. All share their LANGID with the base locale and differ in -> **nothing else** — verified from Access-authored fixtures in `LocaleFixtureCollationProbeTest`, where the -> whole four-byte field is printed raw and reconciled against the parse. +> **nothing else** — verified against Access-authored databases, reading the whole four-byte field raw. A +> reader that takes only the LANGID reads Hungarian Technical as plain Hungarian. > -> > This byte was documented here for a long time as *"`0` in every file observed… if a database ever carries -> > a non-zero `0x0D`, we are truncating a wider value"*. That is exactly what happened, and until the -> > fixtures arrived LibRed read Hungarian Technical as plain Hungarian. Kept as a note because the warning -> > paid for itself: the same reasoning applies to any field we observe as constant-zero. -> -> **Verified** against databases authored with each order (`México`/`O'Brien`/`a`/`A` fixtures): a -> v1 text column has `0x0E = 01` and produces index keys unlike the v0 encoder. (Reading the *version* was -> once a real bug too: LibRed read the byte at `0x0D` — `0` in both General orders — and so reported every -> database as v0.) +> **Verified** against databases authored with each order: a v1 text column has `0x0E = 01` and produces +> index keys unlike the v0 encoder. The version is the byte at `0x0E`, not `0x0D` — `0x0D` is `0` in both +> General orders, so reading it reports every database as v0. > > **The collation is stored in two places, and they agree.** The `(LCID, version)` sort order lives > *both* per column (here: locale at `0x0B`–`0x0C`, version at `0x0E`) *and* database-wide in the > obfuscated page-0 header (LCID at `0x6E`, version byte at `0x71`; see > [page-00-database.md §2.1](page-00-database.md#21-the-obfuscated-header-0x180x98)). For a given -> database the per-column `0x0E` equals the page-0 `0x71` (verified across seven files). Changing the +> database the per-column `0x0E` equals the page-0 `0x71` (verified). Changing the > default **language** moves the LCID in both places; flipping **General vs General Legacy** moves the > version byte in both places. > -> (An earlier revision claimed page 0 held *no* sort-order value — that was inferred from a v0/v1 diff -> whose obfuscated `0x71` looked like creation-date noise. De-obfuscating the header with the fixed mask -> showed the version is there too.) +> (In a raw v0/v1 diff the still-obfuscated `0x71` looks like creation-date noise; de-obfuscate the header +> with the fixed mask before reading it.) > **The format allows a per-column collation; the engine never writes one.** Each descriptor carries its > own four bytes, so a file *could* hold a column sorting differently from its database — and one can be @@ -217,15 +204,14 @@ as 20 bytes of raw UTF-16LE. > locale argument of the **CreateDatabase** method … or the **CompactDatabase** method", and says outright > that *"you can't set a collating order for an individual index — you can only set it for an entire > table"*. ACE's SQL has no syntax for it and Access's UI offers only the database-wide setting. Hence the -> per-column `0x0E` equalling page-0 `0x71` in every file measured: not a coincidence, an absence of any way -> to make it otherwise. The room is reserved in the format if a future version wants it. +> per-column `0x0E` equalling page-0 `0x71`: not a coincidence, an absence of any way to make it otherwise. +> The room is reserved in the format if a future version wants it. > -> That absence is what makes a **stamped** file the only way to measure a version-1 collation, and it is -> load-bearing for `CollationV1PatchProbeTests`: create the database as General v1, write the target LCID -> onto one column's descriptor, and ACE indexes with it. Which is also why LibRed does **not** implement -> ACE's fallback for a version-1 declaration with no version-1 table (region-specific LCIDs resolve to plain -> General v0, neutral ones to their own language's v0 order — measured across all 404 known LCIDs). It is -> real behaviour on input that no supported tool can produce. +> ACE does honour a **stamped** column: with the database created as General v1 and a target LCID written +> onto one column's descriptor, ACE indexes that column with it. For a version-1 declaration whose LCID has +> no version-1 table, ACE falls back — region-specific LCIDs resolve to plain General v0, neutral ones to +> their own language's v0 order (measured across all known LCIDs). LibRed does **not** implement that +> fallback: it is real behaviour, but on input that no supported tool can produce. > > **Format-version coupling.** Access sets the file format to the lowest version that supports the features > used, so choosing General Legacy in the UI *downgrades the file to the 2007 format*, while General (v1) @@ -240,8 +226,8 @@ as 20 bytes of raw UTF-16LE. > is byte-identical for General legacy (verified). Which of these collations LibRed can **encode index keys > for** is not this section's subject and is answered in [§10.4](page-03-04-index-btree.md): the gate is > `Collation.IsIndexKeyEncodable`, it is deliberately default-closed, and both sort-order versions plus -> several hundred locales now pass it. -> +> several hundred locales pass it. + Variable-length columns carry a *variable index* — their slot in the row's variable-offset table ([§5](page-01-data-and-rows.md)), stored in the descriptor at `0x07`. For an untouched table it equals the column's rank among the variable columns ordered by ascending id. @@ -255,8 +241,8 @@ column's rank among the variable columns ordered by ascending id. #### Declared width limits Two limits bind a declaration, both enforced by ACE when it **opens the file**, so writing past either -damages the database rather than just the table. Measured 2026-09-06 against ACE 16 (OLE DB); LibRed -applies both in `Catalog/RecordLayout.cs`, on create and on every incremental path. +damages the database rather than just the table. Verified against ACE 16 (OLE DB); LibRed applies both in +`Catalog/RecordLayout.cs`, on create and on every incremental path. **Per field: 510 bytes** — 255 Text characters, or 510 bytes of Binary, fixed or variable alike. ACE refuses a wider column through its own DDL identically on `CREATE TABLE`, `ALTER COLUMN` and `ADD COLUMN` @@ -277,8 +263,7 @@ declared width — which is why an all-Text table is unconstrained while a wide 4-byte allowance for a table with no variable columns at all is ACE's; LibRed's own encoder omits the variable section entirely in that case (§5), so its rows come in 4 bytes under what ACE reserves. -Measured on both sides of the boundary at column counts chosen so the null-bitmap term differs by 31 bytes, -and the two boundaries duly differ by 31: +Verified on both sides of the boundary; the null-bitmap term moves the boundary byte for byte: | shape | fixed bytes | widest record | ACE | | --- | ---: | ---: | --- | @@ -291,7 +276,7 @@ and the two boundaries duly differ by 31: ACE's own SQL DDL cannot easily reach this: it declares `GUID` columns **variable** (see [data-types.md](data-types.md)), so a table wide enough to trip the limit takes deliberately wide fixed -columns. `ColumnWidthLimitAccessTests` holds the measurements. +columns. ### 3.8 In-place column type/length change (`ALTER COLUMN`) — verified byte-for-byte @@ -299,10 +284,9 @@ columns. `ColumnWidthLimitAccessTests` holds the measurements. Changing a column's **type or length** does **not** edit that column in place. Access **makes a brand-new column that keeps the old one's ordinal position but takes a fresh id**, copies + converts the data into it, and **leaves the old column's storage as dead space** (it is *not* compacted away). Verified by -diffing whole files before/after `ALTER COLUMN` over the ACE OLE DB provider; LibRed's in-place path reproduces every byte -(`AceModifyByteDiffProbe.Libred_in_place_modify_matches_ace_whole_file`, a theory over fixed / variable / -fixed↔variable / PK / indexed / multi-page / decimal shapes, plus a 20-column 7-step non-sequential stress -run). This is **the same mechanism for every type/length change** — including a *widening* `TEXT(n)→TEXT(m)`; +whole-file diff against ACE; LibRed's in-place path reproduces every byte across fixed, variable, +fixed↔variable, PK, indexed, multi-page and decimal shapes, including repeated non-sequential modifies. +This is **the same mechanism for every type/length change** — including a *widening* `TEXT(n)→TEXT(m)`; there is no cheap "just bump the length" path, ACE burns the id there too. LibRed's Memo/OLE logical rebuild has a different layout but enforces the same id high-water limit; @@ -318,7 +302,7 @@ change **to a variable type**, the variable-column count (`0x2B`) also bumps **+ permanent: repeated modifies keep consuming ids from `0x29`, which is why a heavily-altered table can hit "Too many fields defined" with far fewer than 255 *live* columns (only a compact renumbers). -Probed directly: a 4-column table `A,B,C,D` (ids 0,1,2,3); `ALTER COLUMN B …` → B stays at index 1 with +Example: a 4-column table `A,B,C,D` (ids 0,1,2,3); `ALTER COLUMN B …` → B stays at index 1 with **id 4**; a later `ALTER COLUMN C …` → C stays at index 2 with **id 5**; every *other* column keeps its id. So after modifies the ids are non-contiguous while the physical order is unchanged — which is why the row's null bitmap is keyed by **id** and not by position (§5). @@ -326,7 +310,7 @@ null bitmap is keyed by **id** and not by position (§5). > **An identity ALTER is not always free, and nullability never enters into it.** An identical > `SHORT`/`LONG` declaration consumes no id, even at 255; an identical Memo/OLE declaration *does* consume > one and is rejected at 255. ACE accepts the identity ALTER at 255 for a `NOT NULL` column as readily as a -> nullable one (`ColumnIdBoundaryAccessTests.Ace_identity_alter_at_exhaustion_ignores_nullability`), so +> nullable one, so > LibRed must not compare nullability when deciding an ALTER is a no-op — and no ALTER path carries it > anyway, since `Required` is applied separately and `RewriteColumn` discards the spec's value. @@ -362,13 +346,14 @@ variable chunk at variable-index = the old var count. The leading count, variabl and null bitmap are then rebuilt per §5 (count and bitmap width = max id + 1, dead ids' bits set present). **Indexed target — full index rebuild.** When the modified column is in an index, ACE reconstructs that -index (its keys change type). Verified reproduction: +index (its keys change type). Verified: - Allocate a **fresh empty root leaf** (an appended page); the old root is freed **last** (so the new root gets the appended page, not the recycled old one). - **Re-point the index-data block** (§3.5) in the TDEF: the target's **burned id** replaces the old id in - its column slot (`0x04` array), the **new root** at `0x26`, the **new usage-map row** at `0x22`, and the - index **stats block** (§3.3.1) first word bumped **0→1**. + its column slot (`0x04` array), the **new root** at `0x26`, and the **new usage-map row** at `0x22`. The + index's **stats block** (§3.3.1) is set from the rebuilt index: total = its entries, unique = its distinct + keys among the rows present. - **Recycle the owned-pages usage-map row** the way ACE does — the append/move/tombstone dance, and the stale bytes it deliberately leaves behind, are [page-05 §9](page-05-usage-maps.md). - **Back-fill** the new B-tree with new-type keys (one `AddEntry` per row). diff --git a/src/LibRed/docs/format/page-02c-default-values.md b/src/LibRed/docs/format/page-02c-default-values.md index c1b2d60e6..3d8429f6f 100644 --- a/src/LibRed/docs/format/page-02c-default-values.md +++ b/src/LibRed/docs/format/page-02c-default-values.md @@ -5,11 +5,10 @@ > and the function catalog in [`../functions.md`](../functions.md). Reference for how a column's `DEFAULT` behaves in the Jet/ACE engine and how LibRed matches it. Everything here -is **verified against ACE** (probed via `Microsoft.ACE.OLEDB.16.0`) and, where noted, corroborated by the DAO +is **verified against ACE** (`Microsoft.ACE.OLEDB.16.0`) and, where noted, corroborated by the DAO [`Field2.DefaultValue`](https://learn.microsoft.com/office/client-developer/access/desktop-database-reference/field2-defaultvalue-property-dao) -reference. Companion tests live in `LibRed.Engine.Tests` (engine semantics) and `LibRed.Core.Tests` (ACE -round-trips); the on-disk storage of the default text is covered in -[`system-catalog.md`](system-catalog.md) (LvProp / `DefaultValue`). +reference. The on-disk storage of the default text is covered in [`system-catalog.md`](system-catalog.md) +(LvProp / `DefaultValue`). ## The mental model: more than a constant, less than a computed value @@ -64,7 +63,7 @@ capabilities. > **They do not even share a lexer.** ACE's SQL parser accepts `` `backtick` `` identifier quoting — it is > what EF Core's Jet provider emits for every identifier — and the expression service does **not**. It reads > `` `Qty` `` as a field whose name literally includes the backticks and fails with *"Could not find field -> '`Qty`'"*. Measured in two unrelated features: a calculated column's `Expression`, and a `CHECK` constraint, +> '`Qty`'"*. It shows in two unrelated features: a calculated column's `Expression`, and a `CHECK` constraint, > where `ALTER TABLE … CHECK (`Qty` > 0)` is **accepted and stored** and every later INSERT is then refused. > That is the same smuggling route as the compound defaults above, but it lands the other way round: the DDL > parser stores something the service cannot evaluate, so the table ends up unusable rather than usefully @@ -105,13 +104,14 @@ cannot be smuggled past it: a column-ref default written to `LvProp` opens clean ## Functions in a default A default may call any of LibRed's supported scalar functions — the full catalog (with the JES-vs-Access -two-services distinction, the `$`/`B`/`W` variants, and the aggregate set) now lives in its own page: +two-services distinction, the `$`/`B`/`W` variants, and the aggregate set) lives in its own page: **[functions.md](../functions.md)**. Defaults are the *narrowest* place functions are used; the same evaluator serves `SELECT` / `WHERE` / `ORDER BY` / `CHECK`. Default-specific points: `GenUniqueID()` / `GenGUID()` are valid **only** as a default (ACE rejects them in a `SELECT`); the niladic `Now` works bare; and Access forbids a handful of categories in a default even though they are otherwise valid expressions (see the table below). + ## LibRed specifics - **One evaluator.** Defaults, `SELECT` projections, `WHERE`, and `ORDER BY` share `ExpressionEvaluator`. Adding @@ -127,10 +127,10 @@ they are otherwise valid expressions (see the table below). ### Parity of the forbidden categories -| Category | ACE | LibRed | Test | -|---|---|---|---| -| Column reference | reject | reject ("Column not found") | `DateTimeDefaultTests`, `AceSmuggledColRefDefaultTests` | -| Table / query (subquery) | reject | reject (parse) | `DateTimeDefaultTests` | -| SQL aggregate (`Sum`/`Count`) | reject | reject | `AceDefaultExpressionLimitsTests`, `DateTimeDefaultTests` | -| Domain aggregate (`DCount`) | reject | reject | same | -| Unknown / user-defined function | reject | reject ("not supported") | — | +| Category | ACE | LibRed | +|---|---|---| +| Column reference | reject | reject ("Column not found") | +| Table / query (subquery) | reject | reject (parse) | +| SQL aggregate (`Sum`/`Count`) | reject | reject | +| Domain aggregate (`DCount`) | reject | reject | +| Unknown / user-defined function | reject | reject ("not supported") | diff --git a/src/LibRed/docs/format/page-02d-constraints.md b/src/LibRed/docs/format/page-02d-constraints.md index ee45a4ef5..a6bd73442 100644 --- a/src/LibRed/docs/format/page-02d-constraints.md +++ b/src/LibRed/docs/format/page-02d-constraints.md @@ -9,30 +9,42 @@ statistics: | Offset | Size | Meaning | | --- | --- | --- | -| `0x00` | 4 | Total entry count — **compact-time only** (see note): the row count in a *saved/compacted* file, but `0` in a live-edited one | -| `0x04` | 4 | **Unique entry count** — distinct entries ever added; maintained live on every insert (see note) | +| `0x00` | 4 | Total entry count — written only when the index is **built** or the file compacted (see note), not on insert | +| `0x04` | 4 | **Unique entry count** — maintained live on insert, set when the index is built (see note) | | `0x08` | 4 | Reserved (zero observed) | -**These two fields are maintained very differently — verified with an ACE -insert/delete/insert sequence and against saved Northwind tables:** - -- **Total entry count (`+0`) is *not* maintained on insert.** Access leaves it `0` through live - inserts and only writes the row count on **compact/repair**. Saved Northwind tables read - `total == rowCount` (Categories 8, Orders 830, Order Details 2155) precisely because they were - compacted; a freshly SQL-inserted table reads `total == 0` while `rowCount` climbs. A writer - should therefore **leave `+0` at `0`** on insert (LibRed does), not set it to the row count — - doing so would falsely mark the file as compacted. One exception, measured: ACE's **index rebuild** - does bump `+0` (observed 0 → 1), and LibRed reproduces that in `TableCreator` — so "leave it at 0" - is a rule about the INSERT path, not about every writer. -- **Unique entry count (`+4`) *is* maintained live and is cumulative** — Access increments it per - insert and **never decrements** it. Verified: after 3 inserts it is `3`; after deleting a row it - stays `3` (not decremented); after one more insert it is `4`. It equals the current - distinct-value count only with no deletions. A **unique** index gains one distinct key per row, - so a writer increments `+4` by one per insert per unique index (LibRed does this in - `RowInserter`). A **non-unique** index should advance `+4` only when the inserted key is - genuinely new — not yet handled (see the `TODO(non-unique-index-stats)` marker). This applies to - **most** indexes LibRed writes, not none: every FK backing index is non-unique, as is any - `CREATE INDEX` without `UNIQUE`. LibRed exposes `+4` as `IndexDef.UniqueEntryCount`. +**These two fields are maintained very differently (verified vs ACE):** + +- **Total entry count (`+0`) is *not* maintained on insert.** Access leaves it unchanged through live + inserts, deletes and updates. It is written when the index is **built over the rows present** — + `CREATE INDEX` (unique or not), a foreign key's backing index, and the rebuild of an index whose column an + `ALTER COLUMN` changes — to the **number of entries the index then holds** (the rows, less those an + IGNORE NULL index leaves out); and on **compact/repair**, to the row count. So an index created on an + empty table and then filled by inserts reads `0`, while one created over six rows reads `6`. A writer must + **not** set it on insert: that would falsely mark the file as compacted. +- **Unique entry count (`+4`) *is* maintained live and is cumulative** — Access advances it on + **INSERT only**, by one when the row brings a key the index **does not hold at that moment**, and **never + decrements** it. Verified: after 3 inserts into a unique index it is `3`; after deleting a row it stays + `3`; after one more insert it is `4`. It equals the current distinct-key count only with no deletions. + - A **unique** index gains a new key with every row, so it advances on every insert. + - A **non-unique** index advances only for a key not already in it. Keys are compared as index keys, so + collation-equal text (`'a'`, `'A'`) is one key; a **Null is a key** like any other (a second Null adds + nothing), except in an **IGNORE NULL** index, which does not hold it and so never counts it. A key whose + last row was deleted **counts again** when it returns. A multi-column index compares the whole tuple. + - An **UPDATE never advances it**, even one that gives a row a key no other row has. + - It is **one count per real index**: a relationship's logical index sharing a real index (a parent's + primary key) does not advance it a second time. + - **Building the index** — the same builds as `+0` above — sets it to the index's **distinct keys among + the rows present**, discarding any earlier history: after two deletions, rebuilding an index whose + cumulative count was `4` over rows holding `5, 6, 6, 8` gives `3`. + - **Nothing else changes either count**: `ADD`/`DROP COLUMN`, `DROP INDEX` (the other blocks keep their + values), and an `ALTER COLUMN` of a column no index covers all leave every block as it was. The same holds + for a retype to or from Memo/OLE: only an index over the changed column is rebuilt — a primary key + included — and a table referencing this one keeps its foreign-key index's counts. + + LibRed maintains both this way — inserts in `RowInserter`, builds in `TableCreator`'s index back-fill, and + the Memo/OLE retype (which LibRed does by rebuilding the whole table) restores every other index's counts + afterwards — and exposes `+4` as `IndexDef.UniqueEntryCount`. ### 3.5 Index-data block (52 bytes) @@ -45,21 +57,18 @@ insert/delete/insert sequence and against saved Northwind tables:** | `0x23` | 3 | Usage-map page | | `0x26` | 4 | **B-tree root page** | | `0x2A` | 4 | Unknown / reserved (zero observed). mdbtools places a 1-byte index-flags field at `+0x2A`, but ACE's effective flags are at `0x2E` and this is zero in every file checked | -| `0x2E` | 2 | Flags: `0x01` unique, `0x02` ignore-nulls (`WITH IGNORE NULL` — null-keyed rows excluded from the index), `0x08` required (`WITH DISALLOW NULL` / part of a primary key), `0x80` always-set (Access 2000+). Verified vs ACE: a plain index is `0x0080`, `IGNORE NULL` `0x0082`, `DISALLOW NULL` `0x0088`, a PK `0x0089`. | +| `0x2E` | 2 | Flags: `0x01` unique, `0x02` ignore-nulls (`WITH IGNORE NULL` — null-keyed rows excluded from the index), `0x08` required (`WITH DISALLOW NULL` / part of a primary key), `0x80` always-set (Access 2000+). Verified vs ACE: a plain index is `0x0080`, `IGNORE NULL` `0x0082`, `DISALLOW NULL` `0x0088`, a PK `0x0089`. There is **no clustered flag**: `CLUSTERED`/`NONCLUSTERED` after `PRIMARY KEY` or `UNIQUE` in a constraint is accepted and stores nothing (the file is byte-identical without it), and DAO's `Index.Clustered` reads `False` even on an index created with it set. | | `0x30` | 4 | Unknown / reserved (zero observed) — trailing bytes of the 52-byte block | -> **The 10-column cap must be enforced on the incremental path too** — the same lesson as the 32-index cap -> below, and missed the same way. `TdefBuilder` rejects an over-wide index when a table is created with its -> indexes, but `CREATE INDEX` and `ADD FOREIGN KEY` on an existing table go through -> `TableCreator.InsertIndex`, which checked nothing: the block builder filled its ten slots and marked the -> rest unused, so LibRed accepted an 11-column index and stored a 10-column one. ACE refuses outright — -> *"Cannot have more than 10 fields in an index."* +> **The 10-column cap must be enforced on the incremental path too**, as must the 32-index cap below. +> `TdefBuilder` rejects an over-wide index when a table is created with its indexes, but `CREATE INDEX` and +> `ADD FOREIGN KEY` on an existing table go through `TableCreator.InsertIndex`. A block builder that fills +> its ten slots and marks the rest unused silently stores a 10-column index for an 11-column request. ACE +> refuses outright — *"Cannot have more than 10 fields in an index."* > -> That failure was quieter than the other overruns, and worse for it. Too many indexes yields a file Access -> cannot open, and an over-long record yields a row it cannot read; this yielded a file ACE reads happily, -> holding an index over different columns from the ones requested. It was inconsistent internally as well — -> the duplicate-key scan validated against all eleven requested columns while the back-fill populated the -> index from the ten the TDEF recorded (`IndexColumnCountAccessTests`). +> That failure is quieter than the other overruns, and worse for it. Too many indexes yields a file Access +> cannot open, and an over-long record yields a row it cannot read; a truncated index yields a file ACE +> reads happily, holding an index over different columns from the ones requested. > **Unique (`0x01`) enforcement treats NULLs as distinct (verified vs ACE).** A `UNIQUE` index (that is > **not** `WITH IGNORE NULL`) rejects a duplicate **non-null** key but permits **multiple NULL** keys — two @@ -71,6 +80,15 @@ insert/delete/insert sequence and against saved Northwind tables:** > (`0x02`) additionally leaves null-keyed rows out of the B-tree entirely; a PK (`0x08` required) forbids > nulls, so the question doesn't arise. +> **Required (`0x08`) and the primary key over existing rows (verified vs ACE).** A table has **one** primary +> key: adding a second — by `ALTER TABLE … ADD CONSTRAINT … PRIMARY KEY` or `ADD COLUMN … PRIMARY KEY` — is +> refused with *"Primary key already exists."* A required index (a primary key, by any of the three ways of +> adding one, or `WITH DISALLOW NULL`) over rows that already exist is refused when a row has a NULL key — +> *"Index or primary key cannot contain a Null value."* That is what an `ADD COLUMN … PRIMARY KEY` on a table +> holding rows meets, since the new column is NULL on every old row; the exception is an AutoNumber column, +> which numbers the old rows as it is added (§3.1) and so takes the key. A `WITH IGNORE NULL` +> index is not required and is accepted over the same rows. + A table has **at most 32 index-data blocks** (the `0x33` count, §3.1) — the Jet/ACE "32 indexes per table" limit, counting the indexes that back primary keys, unique constraints and the child side of relationships. Incoming relationships add *logical* index-info blocks (§3.6) that reuse an existing data @@ -99,34 +117,24 @@ Because a data block must be named by a logical block, `0x33 ≤ 0x2F` always ho binding constraint and `0x33` derivable from it. It is also why the physical cap cannot be reached in isolation: 33 plain indexes push both counts to 33 together. -> **Measured 2026-09-06 — a file LibRed wrote that Access cannot read.** Creating EF Core's -> `ComplexNavigationsSharedType` model, `Level1` ended at **46 logical blocks against 31 data blocks, with -> a continuation page**. ACE refuses to build the same model at all — *"There are too many indexes on table -> 'Level1'. Delete some of the indexes on the table and try the operation again."* -> -> Opening the resulting file in Access logs `-1206 Unrecognized database format` plus fifteen -> `-1305 … could not find the object 'Level1'` into `MSysCompactError`, and the table is absent from the -> object list. Nothing in that failure names indexes or a limit. LibRed reads the same file back without -> complaint. +> **Overrunning the logical count writes a file Access cannot read.** ACE's own DDL refuses such a table — +> *"There are too many indexes on table '…'. Delete some of the indexes on the table and try the operation +> again."* A file written with one anyway makes Access log `-1206 Unrecognized database format` plus +> `-1305 … could not find the object '…'` entries into `MSysCompactError`, and the table is absent from the +> object list. Nothing in that failure names indexes or a limit. > -> **The boundary is exactly 32, isolated against ACE.** `Level1` carried two anomalies at once — the logical -> count *and* the only multi-page TDEF in the file — so it could not settle which mattered. A pair of -> minimal tables did (`IndexCountLimitAccessTests`): one primary key plus incoming relationships, one data -> block, single page, differing only in the count. +> **The boundary is exactly 32, on the logical count alone** (verified vs ACE: one primary key plus incoming +> relationships, one data block, single page, differing only in the count): > > | `0x2F` | `0x33` | continuation | ACE | > | --- | --- | --- | --- | > | 32 | 1 | none | reads the table | > | 33 | 1 | none | refuses it | > -> So the logical count alone does it, at exactly the same 32 as `0x33`, and the continuation page on -> `Level1` was incidental. Note a logical count merely *exceeding* the data count is ordinary and harmless — -> `InheritanceOne` in the same file sits at 13 against 5 and reads fine. Only the magnitude matters. -> -> Two practical notes. `0x33` was **31** on `Level1`, one below the cap, so the read-side check on that -> count came within a single index of catching this by luck rather than design. And both counts must be -> validated on the **incremental** write paths, not only at create time: a table built one -> `CREATE INDEX`/`ALTER TABLE` at a time never passes through the whole-table check. +> A continuation page plays no part. A logical count merely *exceeding* the data count is ordinary and +> harmless; only the magnitude matters. Both counts must be validated on the **incremental** write paths, +> not only at create time: a table built one `CREATE INDEX`/`ALTER TABLE` at a time never passes through +> the whole-table check. ### 3.6 Index-info block (28 bytes) — one per *logical* index @@ -148,9 +156,8 @@ The index **name** read at the same ordinal applies to this logical index. To na physical (data-block) index, prefer a real index's name over a foreign-key relationship's (distinguished by `0x11` ≠ 0), and take `IsPrimaryKey` from the type byte `0x17`. -> **Writing a relationship's logical blocks — verified by having ACE create a minimal `P1(Id PK)` / -> `C1(Id PK, Pid FK→P1.Id)` pair and diffing.** The **child** (referencing) table gives its FK-column -> index a *single* logical block that **is** the relationship: `index_num2` → the FK-column data +> **Writing a relationship's logical blocks (verified byte-for-byte vs ACE).** The **child** (referencing) +> table gives its FK-column index a *single* logical block that **is** the relationship: `index_num2` → the FK-column data > block, `0x0C = 0x02` (outgoing), `0x11` = parent page, `0x17 = 0x02`, name = the constraint name. > The **parent** (referenced) table gains an **extra** logical block beyond its data blocks: > `index_num2` → its referenced-key (PK) data block, `0x0C = 0x01` (incoming), `0x11` = child page, @@ -162,9 +169,22 @@ physical (data-block) index, prefer a real index's name over a foreign-key relat > `DROP CONSTRAINT` the count sits below the max and a count-derived number collides with a live block — > leaving two blocks claiming the number each end's `0x0D` cross-link names. > -> The parent key must be a **unique or primary** index over the referenced columns. Measured: over a plain +> The parent key must be a **unique or primary** index over the referenced columns. Over a plain > non-unique index ACE refuses the relationship — *"No unique index found for the referenced field of the > primary table"* — while the same shape over a `PRIMARY KEY` succeeds. +> +> **Each child column must have its parent column's storage type** (verified vs ACE over every pairing of the +> column types). Lengths do not matter: `TEXT(5)`, `TEXT(20)` and `CHAR(10)` pair with one another, `DECIMAL`s +> of any precision and scale pair, and `BINARY` pairs with `VARBINARY`. An AutoNumber is a Long on either side. +> Any other pairing is refused — *"Relationship must be on the same number of fields with the same data types."* +> — as is a column count that differs between the two sides. +> +> **`REFERENCES table` with no column list references the parent's primary key** (verified vs ACE): the child +> columns pair with the key's columns in order, whatever either side's columns are named, and +> `MSysRelationships` records those key columns. The parent must have a primary key — a unique index does not +> stand in for one — or the relationship is refused with *"Cannot create relationship. Referenced table '…' +> does not have a primary key."* In a table referencing itself, `CREATE TABLE` resolves the reference against a +> primary key declared **earlier** in the statement; with the key declared after the reference, it is refused. > Cascade `ON UPDATE`/`ON DELETE` set `0x15`/`0x16` to `0x01` on **both** ends' blocks. > > **Self-reference** (a table whose FK targets itself): both ends live in the **one** TDEF, each with @@ -173,14 +193,13 @@ physical (data-block) index, prefer a real index's name over a foreign-key relat > incoming block is numbered after the data-block logical indexes (`index_num` = data-block count). > Verified byte-for-byte against an ACE-created self-reference. -> **`index_num` (`0x04`) vs `index_num2` (`0x08`) — verified against Northwind.** `0x04` is the +> **`index_num` (`0x04`) vs `index_num2` (`0x08`) (verified).** `0x04` is the > logical index's own unique number; `0x08` is the ordinal of the **real index-data block** (§3.5) > it maps to. They differ because **several logical indexes share one real block**: a relationship > (`0x11` ≠ 0) reuses the real index on *this table's* side of the foreign key rather than owning its -> own. Confirmed on Orders (7 real / 13 logical): real block 3 = `OrderID` `PK_Orders` is referenced -> both by its own PRIMARY logical block (`index_num2 = 3`) and by an **incoming** relationship -> (`index_num2 = 3`, `fkTablePage` = Order Details), while the **outgoing** FK relationships map to -> the child-column real indexes (CustomerID/EmployeeID/ShipVia). So `index_num2` points to the real +> own. A referenced table's primary-key block is named both by its own PRIMARY logical block and by +> each **incoming** relationship (same `index_num2`, `fkTablePage` = the child table), while its +> **outgoing** FK relationships map to the child-column real indexes. So `index_num2` points to the real > index on this table's side — the child FK column for an outgoing FK, the referenced key (PK) for an > incoming one — which is exactly mdbtools' "index into index cols list". LibRed's reader keys off > `0x08` (its `DataNumber`) and does not use `0x04`. diff --git a/src/LibRed/docs/format/page-02e-calculated-columns.md b/src/LibRed/docs/format/page-02e-calculated-columns.md index 7fccc6cc1..a2b6e9a91 100644 --- a/src/LibRed/docs/format/page-02e-calculated-columns.md +++ b/src/LibRed/docs/format/page-02e-calculated-columns.md @@ -6,9 +6,8 @@ describes **engine behaviour** rather than on-disk layout; the 25-byte descripto long-value route live in [page-02b-columns §3.4a](page-02b-columns.md), and the `LvProp` property blob that carries the expression is in [system-catalog.md](system-catalog.md). -Everything here is **measured** — against ACE-authored columns of every type DAO will create -(`CalculatedColumnAccessTests`), against `AdventureWorks_Learn_To_Write_DAX.accdb` from the wild, and from -the write side by having ACE read back what LibRed produced. +Everything here is **measured** — against ACE-authored columns of every type DAO will create, and from the +write side by having ACE read back what LibRed produced. > **There is no single "what ACE does" to match.** Four independent gatekeepers enforce different rules and > do not share a rulebook: **DAO** is the only thing that can author a calculated column and refuses to @@ -43,8 +42,7 @@ extended mode. ## What the file actually carries -Three places, and all three must agree. Measured with DAO-authored columns of every type it will create -(`CalculatedColumnAccessTests`). +Three places, and all three must agree. Measured with DAO-authored columns of every type it will create. **1. The column descriptor** — a *promoted storage type*, the `0xC0` extended flag, always variable-length, declared length `39` for a value type and `0` when the result is a Memo. @@ -70,30 +68,31 @@ descriptor `Double`, `ResultType` `Int32`, and a 4-byte Int32 payload. So: > **descriptor type = promotion of the expression's type · `ResultType` = the declared type · payload is > encoded per `ResultType`.** -**This makes Phase 1 a bug fix, not a refinement.** `CalculatedValue.StoredType` infers the payload type -from the declared type and the payload width, which only works while the two agree. Where they diverge it -is wrong, and measurably so: +**The payload type cannot be inferred from the descriptor type or the payload width.** That only works +while the declared and expression types agree; where they diverge a reader that guesses misreads the value +or fails outright: -| column | descriptor | `ResultType` | payload | LibRed today | +| column | descriptor | `ResultType` | payload | read by guessing | | --- | --- | --- | --- | --- | -| `CDbl([Qty])` declared `dbLong` | Double | Int32 | 4 bytes | **`1E-44`** — reads the Int32 as a Single | -| `CDbl([Qty])` declared `dbCurrency` | Double | Currency | 8 bytes | **`3.45846E-319`** — reads the scaled int64 as a Double | -| `CDbl([Qty])` declared `dbInteger` | Double | Int16 | 2 bytes | **throws** `has 2 bytes; expected 8` | -| `CDbl([Qty])` declared `dbText` | Double | Text | *n* | **throws** | -| `CDbl([Qty])` declared `dbBoolean` | Double | Boolean | 1 byte | **throws** | +| `CDbl([Qty])` declared `dbLong` | Double | Int32 | 4 bytes | **`1E-44`** — the Int32 read as a Single | +| `CDbl([Qty])` declared `dbCurrency` | Double | Currency | 8 bytes | **`3.45846E-319`** — the scaled int64 read as a Double | +| `CDbl([Qty])` declared `dbInteger` | Double | Int16 | 2 bytes | **fails** — 2 bytes where 8 are expected | +| `CDbl([Qty])` declared `dbText` | Double | Text | *n* | **fails** | +| `CDbl([Qty])` declared `dbBoolean` | Double | Boolean | 1 byte | **fails** | -The corpus did not contain such a column, and every DAO-authored case in `CalculatedColumnAccessTests` -has the declared and expression types agreeing, which is why this passed. Reading `ResultType` removes the -guess entirely. +Reading `ResultType` removes the guess entirely. ## The expression language ACE accepts -Enumerated over every scalar function [functions.md](../functions.md) catalogues plus every operator — 147 -candidates, **72 accepted**. This is the whole surface, not a sample. +Enumerated over every Access / VBA scalar function [functions.md](../functions.md) catalogues plus every operator. This is +the whole surface, not a sample. **Operators** — `+` `-` `*` `/` `^` `&`, `+` as concatenation, `=` `<>` `<` `>` `<=` `>=`, `And`, `Or`, `Not`, `Is Null`, `Like`, `In`, unary minus, parentheses, literal-only expressions. +**Text literals** — in double or single quotes, a doubled quote standing for itself: `[A] & "it's"` and +`[A] & 'it''s'` are both accepted, stored as written in `Expression`, and give the same values. + **Math** — `Abs` `Sgn` `Int` `Fix` `Round` `Sqr` `Exp` `Log` `Sin` `Cos` `Tan` `Atn`. **String** — `Len` `LCase` `UCase` `Trim` `Left` `Right` `Mid` `InStr` `Space` `String` `Str` `Asc`. @@ -111,19 +110,16 @@ Everything else is refused, in one of **four** distinct ways — a validator has | Message | Cases | | --- | --- | -| `The expression cannot be used in a calculated column.` | the policy whitelist — 69 cases, listed below | +| `The expression cannot be used in a calculated column.` | the policy whitelist — listed below | | `Expression not supported for conversion` | `Eqv`, `Imp`, `Between` | -| `Syntax error in expression.` | a call with a **missing optional argument** (below), and `CDec`, which the parser does not know at all | +| `Syntax error in expression.` | a call with the **wrong number of arguments** (below), and `CDec` at any arity — the one exception to that reading | | `The expression cannot be saved because it refers to another table.` / `... refers to itself.` | scope violations | ### Cross-checked against Access's own Expression Builder Access's Expression Builder, opened on a calculated column, lists the functions it will offer. Its list and -the 49 functions measured above agree **exactly** — every function the builder shows was found accepted, and -nothing found accepted is missing from it. That is a genuinely independent confirmation: one list comes from -the UI, the other from driving DAO. - -It also exposed two things the sweep never tried, both of which turned out to matter. +the functions measured above agree **exactly** — every function the builder shows is accepted, and nothing +accepted is missing from it. Two entries on it need care. **`IsEmpty` is accepted, and is a constant.** VBA's `IsEmpty` asks whether a Variant was never initialised, which a stored column value never is: ACE returns `False` for a column holding a value *and* for one holding @@ -143,17 +139,15 @@ states it twice: *"Although the third parameter for the Mid method is optional i require you to supply all parameter values"*, and *"you must supply all parameters for methods that you call, even if the parameters are optional."* -It is real, and it is why `InStr` first measured as refused — `InStr([A],"a")` is a **syntax error**, while -`InStr(1,[A],"a")` and `InStr(1,[A],"a",0)` are both **accepted**. `Mid` behaves the same way: `Mid([A],2)` -is a syntax error, `Mid([A],2,3)` is fine, which is the article's own example. A calculated-column parser -therefore has to enforce arity, and must report a wrong count as a *syntax* error rather than a policy +It is real: `InStr([A],"a")` is a **syntax error**, while `InStr(1,[A],"a")` and `InStr(1,[A],"a",0)` are +both **accepted**. `Mid` behaves the same way: `Mid([A],2)` is a syntax error, `Mid([A],2,3)` is fine, +which is the article's own example. A calculated-column parser therefore has to enforce arity, and must report a wrong count as a *syntax* error rather than a policy refusal. > **A syntax error means the wrong ARITY, not an unknown name — the opposite of the obvious reading.** -> Measured with the discriminating pair: `NotAFunction([Qty])`, a name nothing could know, gets the *policy* -> message; `Abs([Qty],2)` and `Abs()`, a whitelisted name with the wrong count, get *"Syntax error in -> expression."* So "syntax error" says nothing about whether ACE knows the function. `CDec` is the one case -> that resists the rule: at one argument it is still a syntax error rather than a policy refusal, which sets +> `NotAFunction([Qty])`, a name nothing could know, gets the *policy* message; `Abs([Qty],2)` and `Abs()`, a +> whitelisted name with the wrong count, get *"Syntax error in expression."* So "syntax error" says nothing +> about whether ACE knows the function. `CDec` is the one case that resists the rule: at one argument it is still a syntax error rather than a policy refusal, which sets > it apart from every other rejected conversion. > > **`InStr` takes three or four arguments here, never two** — its behaviour otherwise is the ordinary one in @@ -165,17 +159,14 @@ But the rule is **not universal**, whatever the article says. `Weekday([D1])`, ` `WeekdayName(1)` and `Round([Price])` are all accepted with their optionals omitted. So it is per-function, and the only safe course is to record the required arity of each whitelisted function by measurement. -Measured for the financial ten as well, each in its shortest legal VBA form and again with every argument -supplied — `Pmt` `FV` `PV` `NPer` `Rate` `IPmt` `PPmt` `DDB` all accept the short form, and `SLN`/`SYD` have -no optionals to omit; all 18 shapes were accepted -(`Records_which_financial_functions_accept_an_omitted_optional`). So on everything measured so far **`Mid` -and `InStr` are the only two functions that demand every argument** — they are the exception to the -article's rule, not an illustration of it. +The financial ten, each in its shortest legal VBA form and again with every argument supplied: `Pmt` `FV` +`PV` `NPer` `Rate` `IPmt` `PPmt` `DDB` all accept the short form, `SLN`/`SYD` have no optionals to omit, and +every shape is accepted. So on everything measured **`Mid` and `InStr` are the only two functions that demand +every argument** — they are the exception to the article's rule, not an illustration of it. Supplying every optional does **not** rescue anything on the policy list: `Replace`, `StrComp`, `InStrRev`, `StrConv`, `Format`, `FormatCurrency`, `FormatNumber`, `FormatPercent`, `FormatDateTime`, `DatePart`, -`DateDiff`, `DLookup`, `DCount`, `Rnd` and `Nz` were all re-tested in full form and are still refused. The -two rules are independent. +`DateDiff`, `DLookup`, `DCount`, `Rnd` and `Nz` are refused in full form too. The two rules are independent. The article also notes that a calculated field **cannot call a user-defined function**, only built-ins — consistent with everything measured, and with the whitelist being fixed rather than extensible. @@ -189,12 +180,11 @@ Refused by policy: `\` `Mod` `Xor`; every conversion except `CDbl` (`CBool` `CBy `CurrentUser` `DLookup` `DCount`; and every aggregate (`Sum` `Count` `Avg` `Min` `Max` `First` `StDev` `Var`). -**Why `CDbl` and nothing else?** The first sweep gave each conversion a declared type that did *not* match -its result, and only `CDbl` happened to match — so the obvious suspicion was that the rule was about -agreement, not about `CDbl`. Re-running each conversion against its **matching** declared type kills that: -`CBool→dbBoolean`, `CByte→dbByte`, `CInt→dbInteger`, `CLng→dbLong`, `CCur→dbCurrency`, `CSng→dbSingle`, -`CDate→dbDate`, `CStr→dbText` are all still refused, and `CDbl` is accepted against `dbDouble`, `dbLong`, -`dbCurrency` and `dbSingle` alike. It really is `CDbl` and only `CDbl`, regardless of the declared type. +**Why `CDbl` and nothing else?** The rule is not about a conversion agreeing with the declared type. Each +conversion against its **matching** declared type — `CBool→dbBoolean`, `CByte→dbByte`, `CInt→dbInteger`, +`CLng→dbLong`, `CCur→dbCurrency`, `CSng→dbSingle`, `CDate→dbDate`, `CStr→dbText` — is refused, and `CDbl` is +accepted against `dbDouble`, `dbLong`, `dbCurrency` and `dbSingle` alike. It really is `CDbl` and only +`CDbl`, regardless of the declared type. The reading most consistent with the evidence — offered as a hypothesis, not a measured fact — is that the evaluator's native numeric type **is** Double (OA's `R8`), so `CDbl` is an identity assertion rather than a @@ -207,7 +197,7 @@ refused — so the curation is still partly historical. "Deterministic, row-local, no I/O" explains the big exclusions — `Rnd`/`Timer`/`Now`/`Date`/`Time` are volatile, `DLookup`/`DCount`/aggregates and other-table references leave the row, `Environ`/`CurDir`/ `CurrentUser` are environment. It does **not** explain the edges, which have to be taken as measured: -`Trim` is in but `LTrim`/`RTrim` are out; `Right` is in but `InStr` is out; `Asc` is in but `Chr` is out; +`Trim` is in but `LTrim`/`RTrim` are out; `InStr` is in but `InStrRev` is out; `Asc` is in but `Chr` is out; `Str` is in but `Val` is out; `CDbl` is in but every other conversion is out; `MonthName`/`WeekdayName` are in but `Format` is out; `Like` and `In` are in but `Between` is out; and all ten financial functions are in. @@ -220,34 +210,35 @@ was chosen over inverting with an interface Engine supplies, because inversion b that needs the answer is the one that cannot reach the implementation, so a Core-only caller (`JetDatabase.Insert` is public API) would still have to refuse. The subset is small, frozen by ACE, and semantically its own language — the Access *expression service*, not Access SQL — and it must match ACE -exactly, including null propagation, Currency scaling and the compressed-vs-raw text choice, so it wants its -own focused tests wherever the code lives. The financial functions and the two name functions are **ported** -from `ExpressionEvaluator` rather than re-derived, so one formula cannot answer differently through SQL than -through a calculated column. Moving the shared primitives down into Core remains the answer if the -duplication starts to bite. +exactly, including null propagation, Currency scaling and the compressed-vs-raw text choice. The financial +functions and the two name functions are **ported** from `ExpressionEvaluator` rather than re-derived, so one +formula cannot answer differently through SQL than through a calculated column. Moving the shared +primitives down into Core remains the answer if the duplication starts to bite. **Authoring.** `ColumnSpec.Calculated` builds one: promoted storage type in the descriptor, `0xC0`, variable slot, the constant declared length. `CREATE TABLE` and `ADD COLUMN` write the `Expression`/`ResultType`/`FCMin*Ver` properties. A Memo result gets its long-value maps, keyed off the -**result** type because the declared type is `Text` and says nothing — which also needed the write-side guard -in `TdefBuilder` relaxed, since it demanded a map's owner be Memo/OLE, the same assumption the reader had -already had to drop. The ACE-14 gate goes through `EnsureFormatAtLeast`, raising the file rather than -refusing, as `BIGINT`/`DATETIME2` already do. +**result** type because the declared type is `Text` and says nothing — so neither the reader nor the +write-side guard in `TdefBuilder` may demand that a map's owner be Memo/OLE. The ACE-14 gate goes through +`EnsureFormatAtLeast`, raising the file rather than refusing, as for `BIGINT`/`DATETIME2`. -**Validation runs before anything is written** (`CalculatedExpression.ParseValidated`): the whitelist below, +**Validation runs before anything is written** (`CalculatedExpression.ParseValidated`): the whitelist above, plus self-reference, an unknown column, and the `$` name variants. It is mandatory rather than a courtesy — an expression ACE rejects does not produce a merely odd file, it produces a column ACE refuses to read at all. Measured end to end: ACE accepts, evaluates and *recomputes* a column LibRed authored, for every result -type including a spilled Memo (`Creates_a_calculated_column_ace_accepts`). +type including a spilled Memo. **`ALTER` is measured, not assumed.** `ADD COLUMN`, `DROP COLUMN`, `ALTER COLUMN` (retype) and -`RenameColumn` all leave a calculated column working — the last only since the reference-repointing fix. +`RenameColumn` all leave a calculated column working. Where an unrelated column's retype requires LibRed's +full table rebuild (for example Memo → Text), the rebuild carries each calculated column's `Expression`/`ResultType` properties along with its descriptor and +regenerates its cached value from the expression; a cached value is not caller-supplied data and cannot be +reinserted verbatim. Dropping a column an expression reads is **refused**, which ACE does not do; see the probe below. **Text compression follows the declared type**, and the encoding rule is in [page-02b-columns §3.4a](page-02b-columns.md) with the rest of the layout. It is called out here because it -is not cosmetic: compressing shrank a 36-character Memo result from 95 bytes to 61, back under the inline -limit, so the value never reached a long-value page at all. +is not cosmetic: compressing a 36-character Memo result shrinks it from 95 bytes to 61, under the inline +limit, so the value never reaches a long-value page at all. ## What the probes settled @@ -264,8 +255,8 @@ reference recomputes, even when the written value is unchanged: | after `UPDATE T SET Qty=7` (referenced, same value) | `14` | | after `UPDATE T SET Qty=10` | `20` | -So Phase 3 must recompute **exactly when a referenced column is in the changed set** — `RowInserter.Update` -already receives that set. Recomputing unconditionally would write bytes ACE would not have written, and +So a writer must recompute **exactly when a referenced column is in the changed set** — `RowInserter.Update` +receives that set. Recomputing unconditionally would write bytes ACE would not have written, and recomputing never would leave a stale cache that ACE would have refreshed. It also means a stale cache is a legitimate on-disk state, so LibRed reading it back verbatim is right. @@ -279,9 +270,8 @@ cached value. **ACE has no route to change an existing calculated column's expression.** DAO refuses to mutate any field of a *saved* `TableDef` — *"Operation is not supported for this type of object."* — and that refusal is -generic, reproduced on an ordinary column as a control, so it says nothing about calculated columns -specifically. Access's own designer must reach it by a path DAO does not expose. There is therefore no -oracle for "alter the expression", and anything LibRed offers there is its own extension rather than +generic: an ordinary column gets it too, so it says nothing about calculated columns specifically. Access's +own designer must reach it by a path DAO does not expose. There is therefore no oracle for "alter the expression", and anything LibRed offers there is its own extension rather than ACE-matching behaviour. **ACE does not protect a reference; every ALTER is accepted, two of them destructively.** Measured through @@ -307,20 +297,18 @@ the column gone one extra statement. An expression that fails to parse counts as throwing: a safeguard must not become a blanket refusal to drop anything because some unrelated column's expression is malformed. -**Renaming a referenced column must repoint the expression, and LibRed did not.** `RenameColumn` renamed the -column and left the expression naming the old one, so ACE failed every read of the calculated column — a -table broken by an operation that named a *different* column, and invisible from LibRed's own side, which -went on reading the stale cache happily. Now fixed -(`CalculatedExpression.RenameColumnReference`, `Libred_alter_keeps_a_calculated_column_working`). The rewrite -is textual so the author's spacing and bracketing survive, which means it has to know where a name is *not* -a reference: inside a string or `#date#` literal, before a `(` as a function name, or as a prefix of a -longer name. - -**A TDEF rebuild keeps the column calculated.** `ADD COLUMN` and `DROP COLUMN` both rebuild the definition, -and the expression lives in the table's property blob rather than the 25-byte descriptor, so there was a -real risk of carrying the `0xC0` flag across while losing the expression — a column ACE could no longer -evaluate. Measured: both preserve `IsCalculated`, the expression and the result type, and ACE still -evaluates the column afterwards (`Rebuilding_a_tdef_preserves_a_calculated_column`). +**Renaming a referenced column must repoint the expression.** A rename that leaves the expression naming the +old column makes ACE fail every read of the calculated column — a table broken by an operation that named a +*different* column, and invisible to a reader that only returns the stale cache. LibRed repoints it +(`CalculatedExpression.RenameColumnReference`). The rewrite is textual so the author's spacing and bracketing +survive, which means it has to know where a name is *not* a reference: inside a string or `#date#` literal, +before a `(` as a function name, or as a prefix of a longer name. + +**A TDEF rebuild must keep the column calculated.** `ADD COLUMN` and `DROP COLUMN` both rebuild the +definition, and the expression lives in the table's property blob rather than the 25-byte descriptor, so a +rebuild that carries the `0xC0` flag across while losing the expression leaves a column ACE can no longer +evaluate. Measured: both preserve `IsCalculated`, the expression and the result type, and ACE still evaluates +the column afterwards. **Indexing one is refused, and that matches Access rather than diverging from it.** Access's *designer* does not offer a calculated column in the Indexes dialog at all, and will not let it be the primary key — the one @@ -333,8 +321,8 @@ every route: `CREATE INDEX`, `CREATE UNIQUE INDEX`, a composite that merely incl **Indexing a calculated column is a trap, not a feature.** ACE accepts `CREATE INDEX` and even `CREATE UNIQUE INDEX` on one — and then refuses every `INSERT` into the table with *"Operation is not supported for this type of object."* Measured for Int16 and Int32 result types, and with the index created -both before and after rows exist. So the index-key-type question is moot: such an index can never be -populated. LibRed should refuse to create one rather than reproduce a state that bricks the table. +both before and after rows exist. So there is no index-key encoding to define: such an index can never be +populated. **Declared length is a constant, and the requested size is discarded.** `39` for every value type, `509` for Text whatever size was asked for (10, 60 and 255 all give 509), `0` plus a long-value map for Memo. `39` is @@ -352,19 +340,6 @@ disagreement about a value nothing ever recomputes. record cap — are accepted both with and without a calculated column on top. The true ceiling was not bracketed, so this rules out a *fixed-region* cost rather than establishing the exact limit. -**The function whitelist is larger than the first sample and arbitrary at the edges.** 81 candidates probed, -43 accepted. Beyond the first sweep, also accepted: `Right`, `LCase`, `Space`, `String`, `Asc`, `Sqr`, -`Exp`, `Log`, `Sin`, `Cos`, `Tan`, `Atn`, `Fix`, `Month`, `Day`, `Hour`, `Minute`, `Second`, `Weekday`, -`DateSerial`, `TimeSerial`, `CDbl`, `Is Null`, `Like`, `In`. Also rejected: `LTrim`, `RTrim`, `StrReverse`, -`InStr`, `InStrRev`, `StrComp`, `Chr`, `Rnd`, `DateValue`, `CDate`, `CInt`, `CCur`, `CBool`, `CVar`, -`IsNumeric`, `IsDate`, `IsError`, `Between`, `Partition`, `Environ`, `CurrentUser`. - -The edges do not follow from the "deterministic, row-local" rule and must be enumerated rather than -reasoned about: **`Trim` is accepted but `LTrim` and `RTrim` are not**; `Right` is accepted but `InStr` is -not; `Asc` is accepted but `Chr` is not; `CDbl` is accepted but every other `C*` conversion is not; `Like` -and `In` are accepted but `Between` is not. `Rnd()` being rejected does fit the rule, as do `Date()` and -`Now()`. - **The engine evaluates the stored text, and enforces the whitelist itself.** Authoring a column with an allowed expression and then patching the stored `Expression` property on the page to another string of the same length — bypassing DAO entirely — shows both halves: @@ -382,23 +357,21 @@ truth for reads. The other two prove the whitelist is **not** a DAO-side courtes the value empty. Two consequences. **Validation is mandatory, not optional** — an unvalidated expression does not produce a -merely odd file, it produces a column ACE cannot read at all. And **LibRed is currently more permissive -than ACE**: it read the stale `14` and a `null` from files ACE refuses outright, because it never looks at -the expression. That is arguably right for a reader, but it is a divergence and should be a decision rather -than an accident. +merely odd file, it produces a column ACE cannot read at all. And **a reader that returns the cached value +is more permissive than ACE**, which refuses such a column outright; whether LibRed should match that is +[Still open](#still-open) item 2. ## Still open -1. ~~**Where `39` and `509` come from**~~ — **as far as measurement can take it** - (`Records_the_declared_length_of_every_calculated_type`). There are **four** constants, not two: `0` for - Memo, `509` for Text at any requested size, **`510` for Binary** — which nothing had ever created, so the - earlier sampling missed it — and `39` for everything else, GUID included. Nothing moves them: not the - requested size, not the length of the expression. *Why* those particular numbers is not something ACE can - be asked; what matters for writing them is that they are fixed, and now they are known to be. -2. **Should LibRed refuse to read what ACE refuses?** Two cases, and they have now been decided differently. - A **cached error** throws, matching ACE, which will not hand back such a row at all — settled, and the - precedent for the rest. An **unwhitelisted expression** is still read: the cached value is perfectly - well-formed, and nothing has been measured that says ACE's refusal there protects anything on disk. +1. ~~**Where `39` and `509` come from**~~ — **as far as measurement can take it.** There are **four** + constants: `0` for Memo, `509` for Text at any requested size, **`510` for Binary**, and `39` for + everything else, GUID included. Nothing moves them: not the requested size, not the length of the + expression. *Why* those particular numbers is not something ACE can be asked; what matters for writing + them is that they are fixed. +2. **Should LibRed refuse to read what ACE refuses?** The two cases are decided differently. A **cached + error** throws, matching ACE, which will not hand back such a row at all — settled, and the precedent for + the rest. An **unwhitelisted expression** is still read: the cached value is perfectly well-formed, and + nothing measured says ACE's refusal there protects anything on disk. The tension between the two — strictness for correctness against leniency for recovery — is better resolved by a **lenient "recovery" mode** than by softening the default. Reading a damaged or @@ -407,18 +380,14 @@ than an accident. engine-wide rather than a calculated-column feature: surface the error rather than throwing on it, keep enumerating past rows that cannot be decoded, and report what was skipped. Noted as a future direction, not scheduled. -3. ~~**`InStr` and `CDec` fail as a *syntax* error**~~ — **answered**, and the guess behind it was wrong: a - syntax error means the wrong arity, not an unknown name. See the note above; the finding also fixed - `InStr`, which LibRed had been refusing at three arguments. -4. ~~**The full ACE value-matrix parity test**~~ — **done** - (`Computes_the_same_values_as_ace_across_the_matrix`). Fifteen expressions over five rows — nulls, the - empty string, zeros, negatives, a tiny Currency, a non-ASCII character and two pre-epoch times — agree - with ACE in 74 of 75 cases. - - The one exception is a **LibRed divergence**, and it is still open. **The VBA conversion functions do not - propagate Null, they raise on it**, so `CDbl(Null)` is an error rather than a Null result. LibRed applies - one blanket rule — any Null argument yields Null — which is right for every other whitelisted function and - wrong for `CDbl`, the only conversion on the list. +3. ~~**`InStr` and `CDec` fail as a *syntax* error**~~ — **answered**: a syntax error means the wrong arity, + not an unknown name. See the note above. +4. ~~**Full value parity with ACE**~~ — **done.** Evaluated values agree with ACE over nulls, the empty + string, zeros, negatives, a tiny Currency, a non-ASCII character and pre-epoch times, with one exception. + + The one exception is `CDbl` over a Null. **The VBA conversion functions do not propagate Null, they raise + on it**, so `CDbl(Null)` is an error rather than a Null result — unlike every other whitelisted function, + which propagates Null. `CDbl` is the only conversion on the list. The bytes settle what "an error" means on disk, and it is **not** the same as Null: the envelope's leading field holds the **VBA error number** (94 "Invalid use of Null", 13 "Type mismatch"), and an error envelope @@ -429,11 +398,11 @@ than an accident. read exactly as before. **Writing** is settled too, and diverges deliberately: **LibRed refuses the row.** ACE accepts the insert - and caches the failure, after which its own reader rejects that row — and the state is **sticky**. - Measured: a compact-and-repair does *not* clear it (still error 94 afterwards, both rows intact); the only - thing that does is an `UPDATE` writing a column the expression reads, forcing a recompute — which needs a - value for the very column that was Null. So writing it produces a row that neither engine can read and - that the obvious repair does not fix. Refusing costs the caller a guard and keeps the file readable, the + and caches the failure, after which its own reader rejects that row — and the state is **sticky**: a + compact-and-repair does *not* clear it (still error 94 afterwards, rows intact); the only thing that does + is an `UPDATE` writing a column the expression reads, forcing a recompute — which needs a value for the + very column that was Null. So writing it produces a row that neither engine can read and that the obvious + repair does not fix. Refusing costs the caller a guard and keeps the file readable, the same trade as refusing to drop a column an expression reads. 5. **Changing an existing column's expression or result type — out of scope by decision, not unfinished.** ACE offers no route to it at all, so LibRed would be *defining* the behaviour rather than matching any, diff --git a/src/LibRed/docs/format/page-03-04-index-btree.md b/src/LibRed/docs/format/page-03-04-index-btree.md index 5f9a92170..8aef118f4 100644 --- a/src/LibRed/docs/format/page-03-04-index-btree.md +++ b/src/LibRed/docs/format/page-03-04-index-btree.md @@ -12,13 +12,13 @@ | `0x01` | 1 | Flags (observed constant `0x01` — verified) | | `0x02` | 2 | Free space | | `0x04` | 4 | Owning table TDEF page | -| `0x08` | 4 | **The 4-byte field Jet4 inserted** right after the owner — purpose unknown, **`0` observed** on every ACE- and LibRed-written index page (Jackcess has no constant for it either). Inserting it here is what pushes prev/next/tail/compress down by 4 vs Jet3 (see the Jet3→Jet4 note under `0x1B`). | -| `0x0C` | 4 | **Previous leaf page** (`0` on the first/leftmost leaf), little-endian. **Verified against ACE:** on an ACE-built split index the higher-key leaf's `0x0C` points back at the lower-key leaf. (An earlier draft mis-placed prev/next at `0x08`/`0x0C` — wrong by 4 bytes; the real insertion is at `0x08`.) | +| `0x08` | 4 | **The 4-byte field Jet4 inserted** right after the owner — purpose unknown, **`0` observed** on every ACE- and LibRed-written index page. Inserting it here is what pushes prev/next/tail/compress down by 4 vs Jet3 (see the Jet3→Jet4 note under `0x1B`). | +| `0x0C` | 4 | **Previous leaf page** (`0` on the first/leftmost leaf), little-endian. **Verified against ACE:** on an ACE-built split index the higher-key leaf's `0x0C` points back at the lower-key leaf. | | `0x10` | 4 | **Next leaf page** (`0` on the last/rightmost leaf), little-endian. **Verified against ACE — and load-bearing:** Access's full-table `COUNT(*)`/scan descends to the leftmost leaf and walks this forward chain. If it is wrong (e.g. `next` written at `0x0C`), Access stops after the first leaf and **silently sees only those rows** — a data-loss/corruption hazard, since it then treats the rest of the table's space as free. LibRed maintains `0x0C`/`0x10` across splits (§10.5). (This is **Jet3's `0x0C` next-pointer shifted +4**; the child-tail that mdbtools lists at `0x10` is the *Jet3* tail position — in Jet4 it too shifted to `0x14`.) | -| `0x14` | 4 | **Child-tail** page (node pages: the rightmost child, referenced by no entry). For Jet4/ACE this offset is **definitive — byte-for-byte verified** (the tail pointer reads correctly here and drives correct multi-level traversal). This is **Jet3's `0x10` tail shifted +4** by the `0x08` insertion, which is exactly why mdbtools (Jet3) documents the tail at `0x10`. | +| `0x14` | 4 | **Child-tail** page (node pages: the rightmost child, referenced by no entry). **Verified** for Jet4/ACE: the tail pointer read here drives correct multi-level traversal. This is **Jet3's `0x10` tail shifted +4** by the `0x08` insertion, which is exactly why mdbtools (Jet3) documents the tail at `0x10`. | | `0x18` | 2 | Compressed-byte count (shared key prefix length, §10.3). Jet3's `0x14`, shifted +4. | -| `0x1A` | 1 | The **1-byte field Jet4 inserted** just before the bitmask. ACE writes `0` on leaves and `1` on the root of a two-level split index, consistent with a **B-tree level/height** — but **only `0` and `1` have been observed** (no 3-level tree was built against ACE, so `2`+ is a guess). **Required only for leaves (verified):** writing `0x01` on a *leaf* makes ACE fail to open the whole database (`"could not find the object 'Databases'"`). **Node value is cosmetic (verified):** an isolation test — correct leaf-chain offsets but node `0x1A=0` *and* nodes prefix-compressed — still gave ACE the right `COUNT`/`SUM` at 700 and 1500 rows, so Access reads a node's tail child regardless. **Jackcess likewise has no offset constant for `0x1A`** (it tells leaf from node by the page-type byte at `0x00`). LibRed still writes the height to match ACE byte-for-byte, but the only hard requirements are the leaf-chain offsets and a *leaf's* `0x1A=0`. | -| `0x1B` | … | Entry-position bitmask. mdbtools **version-labels** this: bitmask at `0x16` (Jet3) / **`0x1B` (Jet4)** — confirming our offset. The `+5` Jet3→Jet4 shift is **fully decomposed**: a **4-byte field inserted at `0x08`** (right after the owner) plus the **1-byte B-tree level at `0x1A`** = `+5`. Everything between — prev/next leaf, child-tail, compressed count — is Jet3's field shifted by 4, with the level accounting for the final `+1`. No unexplained bytes remain in this header. **Corroborated by Jackcess**, whose `JetFormat` constants give (Jet3 → Jet4): prev `8`→`12`, next `12`→`16`, child-tail `16`→`20`, compressed-count `20`→`24`, entry-mask `22`→`27` — i.e. `0x08`/`0x0C`/`0x10`/`0x14`/`0x16` each `+4`, and the mask an extra `+1`. Two independent implementations now agree on the shift; the `0x08` insertion is the only thing that produces it. (The *positions* are ACE-verified and Jackcess-corroborated; a real Jet3 index page would still be the final confirmation that these are the exact bytes Jet3 lacked — notably Jackcess has no constant for the `0x08` field or `0x1A` either.) | +| `0x1A` | 1 | The **1-byte field Jet4 inserted** just before the bitmask. ACE writes `0` on leaves and `1` on the root of a two-level split index, consistent with a **B-tree level/height** — but **only `0` and `1` have been observed** (no 3-level tree was built against ACE, so `2`+ is a guess). **Required only for leaves (verified):** writing `0x01` on a *leaf* makes ACE fail to open the whole database (`"could not find the object 'Databases'"`). **Node value is cosmetic (verified):** with correct leaf-chain offsets, nodes written with `0x1A=0` *and* prefix-compressed still give ACE the right `COUNT`/`SUM`, so Access reads a node's tail child regardless; leaf vs node is told by the page-type byte at `0x00`. LibRed still writes the height to match ACE byte-for-byte, but the only hard requirements are the leaf-chain offsets and a *leaf's* `0x1A=0`. | +| `0x1B` | … | Entry-position bitmask. mdbtools **version-labels** this: bitmask at `0x16` (Jet3) / **`0x1B` (Jet4)**. The `+5` Jet3→Jet4 shift is **fully decomposed**: a **4-byte field inserted at `0x08`** (right after the owner) plus the **1-byte B-tree level at `0x1A`** = `+5`. Everything between is Jet3's field shifted by 4 (Jet3 → Jet4): prev `0x08`→`0x0C`, next `0x0C`→`0x10`, child-tail `0x10`→`0x14`, compressed count `0x14`→`0x18`, and the mask `0x16`→`0x1B`, the level accounting for its extra `+1`. No unexplained bytes remain in this header. (The Jet4 *positions* are ACE-verified; that these are exactly the bytes Jet3 lacks is not yet confirmed against a real Jet3 index page.) | | `0x1E0` | — | Start of entry data | ### 10.2 Entries @@ -45,7 +45,7 @@ Each entry ends with a **4-byte big-endian** trailing pointer: > role; this avoids relying on an earlier descent check if the file changed between reads. Violations > are reported as `InvalidDataException`. -> **How the insert path revalidates.** A leaf rewrite no longer re-reads and re-decodes the page the descent +> **How the insert path revalidates.** A leaf rewrite does not re-read and re-decode the page the descent > just decoded; it takes that parse from the channel's parsed-page cache and copies the entry list before > mutating it (the cached object is shared with every other reader of the file, so it must never be written > through). The guarantee is unchanged: a cached parse survives only while the bytes behind it are untouched — @@ -54,11 +54,9 @@ Each entry ends with a **4-byte big-endian** trailing pointer: > checked before it is mutated. The copy is shallow by design: an entry is an immutable struct referencing its > key, so copying the list shares the key arrays and costs one array of structs rather than one array per entry. > -> A consequence worth knowing when reading write benchmarks: because an overlay page is never served from the -> cache, this saves nothing for a page already written inside the current transaction. Inserts outside a -> transaction, or early in one, gain the most; a long transaction rewriting the same leaf repeatedly gains -> nothing (measured: a raw insert went 234.6 µs → 164.5 µs, while the same insert inside a transaction did not -> move). +> Because an overlay page is never served from the cache, this saves nothing for a page already written inside +> the current transaction. Inserts outside a transaction, or early in one, gain the most; a long transaction +> rewriting the same leaf repeatedly gains nothing. ### 10.3 Prefix compression @@ -66,9 +64,8 @@ Entries on a page share a leading prefix of `compressedByteCount` (`0x18`) bytes stored in full; its first `compressedByteCount` bytes are the shared prefix, which every subsequent entry omits. Reconstruct: `fullEntry = prefix ++ stored`. -> **The prefix covers the entry whole — it can reach into the trailer.** An earlier revision of this section -> claimed "the trailing pointer is never compressed, so reading row pointers needs none of this". That is -> **wrong**, and it made LibRed reject pages ACE had written. When many rows share a key they are also +> **The prefix covers the entry whole — it can reach into the trailer.** A reader that assumes the trailing +> pointer is never compressed rejects pages ACE writes. When many rows share a key they are also > consecutive on one data page, so the trailer's leading bytes are common too and ACE compresses them away. > A leaf holding 500 rows all keyed `"same"`: > @@ -83,26 +80,25 @@ omits. Reconstruct: `fullEntry = prefix ++ stored`. > leaving two stored bytes per entry. So **size limits apply to the reconstructed entry, never to what is > stored** — a stored entry may be shorter than the 4-byte trailer, and the key may be empty. Take both the > key and the trailer from the reconstruction. Likewise `compressedByteCount` is bounded by the first -> entry's **whole** length, not by its key. (`DuplicateIndexKeyProbeTest`; the old reading refused any index -> with ~500+ equal keys, which is ordinary for a non-unique index.) +> entry's **whole** length, not by its key. A reader that bounds it by the key refuses any index with ~500 or +> more equal keys, which is ordinary for a non-unique index. > **Compression is optional on leaves.** A `compressedByteCount` of 0 (every entry stored in full) > is a valid *leaf* that Access reads without complaint — verified by rewriting a leaf uncompressed > and re-seeking it. **On node (`0x03`) pages, ACE writes them uncompressed (`0x18 = 0`)**, and -> LibRed matches that. *Verified cosmetic:* compressing a node does **not** break Access — an isolation test -> (correct leaf-chain offsets, but nodes compressed and `0x1A=0`) still gave the right `COUNT`/`SUM`. -> The earlier "compression breaks tail descent" idea was a misattribution to the leaf-chain bug (§10.1). -> LibRed writes nodes uncompressed only to stay byte-faithful with ACE, not because it's required. +> LibRed matches that. *Verified cosmetic:* compressing a node does **not** break Access — with correct +> leaf-chain offsets, nodes compressed and `0x1A=0` still give the right `COUNT`/`SUM`. A broken tail descent +> points at the leaf-chain offsets (§10.1), not at node compression. LibRed writes nodes uncompressed only to +> stay byte-faithful with ACE, not because it's required. > > **A leaf with ≤ 1 entry writes `0x18 = 0`.** Prefix compression describes a prefix *shared across > entries*, so with zero or one entry there is nothing to share — ACE writes `compressedByteCount = 0`, -> not the sole key's whole length. (LibRed had a bug writing the full length there via -> `CommonPrefixLength(key, key)`; now `entries.Count ≤ 1 ⇒ 0`. Verified vs ACE on an index whose fresh -> root leaf holds a single key, e.g. the rebuild in §3.8.) +> not the sole key's whole length (which is what a naive `CommonPrefixLength(key, key)` gives). Verified vs +> ACE on a fresh root leaf holding a single key (e.g. the rebuild in §3.8). > > **A leaf is compressed only when it fills, and split only when compressing is not enough.** The prefix is > not a property recomputed on every write; it is applied once, in place, at the moment the page can no -> longer take the next entry. Watching a sequential load one batch at a time shows the cycle twice: +> longer take the next entry. A sequential load runs the cycle twice: > > | after | page | prefix | entries | free | > | ---: | --- | ---: | ---: | ---: | @@ -124,11 +120,9 @@ omits. Reconstruct: `fullEntry = prefix ++ stored`. > still in its uncompressed phase. Descending and random loads read `3, 3, 3, 3`, because a middle split > computes the prefix for both halves as it writes them. > -> Two wrong readings preceded this, both from end state rather than transitions. "LibRed compresses where -> ACE does not" came from a single leaf that had never filled. "ACE recomputes on split and keeps the value -> while appending" was implemented and falsified: pages then fill uncompressed and split without ever being -> compressed, giving 4 leaves and 11,820 bytes against ACE's 3 and 11,334. `IndexSplitPackingAccessTests` -> asserts the result. (The transition probe that produced the measurement was not kept.) +> End state alone misleads: a leaf that has never filled is uncompressed under ACE too. And recomputing the +> prefix on split while keeping it on append is wrong — pages then fill uncompressed and split without ever +> being compressed, giving 4 leaves and 11,820 bytes against ACE's 3 and 11,334. ### 10.4 Key encoding (order-preserving) @@ -158,9 +152,7 @@ Then the value, transformed: - **Integers** (Int16/Int32) and **Currency** (int64): big-endian, with the **sign bit of the first byte flipped**. Descending additionally inverts all bytes. (Decode reverses this.) - **Byte**: the raw byte, with **no** sign flip — Jet's `BYTE` is unsigned, so flipping would sort 128–255 - before 0–127. Verified against ACE over 0, 1, 127, 128, 200 and 255 on an indexed `BYTE` column: LibRed's - keys match ACE's exactly. (This entry previously listed Byte alongside Int16/Int32 as sign-flipped, which - was wrong.) + before 0–127. Verified against ACE on an indexed `BYTE` column, values either side of 128. - **Single / Double / DateTime** (IEEE): if non-negative, flip the first bit; if negative, invert all bytes (ascending). Decode: first byte's top bit set ⇒ was positive (un-flip); else ⇒ was negative (invert all). DateTime is the resulting double via the OLE epoch. @@ -171,14 +163,18 @@ Then the value, transformed: order therefore equals numeric order: negatives (`0x00`) precede non-negatives (`0xFF`), and complementing makes a larger magnitude sort earlier among the negatives. **Zero encodes as positive.** Descending inverts all bytes as usual. Verified byte-for-byte vs ACE, ascending and - descending (`DecimalKeyEncodingTests`); e.g. at scale 4, `1.0` → `7F FF 00…002710` (10000) and + descending; e.g. at scale 4, `1.0` → `7F FF 00…002710` (10000) and `-1.0` → `7F 00 FF…FFD8EF` (`~10000`). - **Boolean:** no flag byte — a single constant: ascending `0x00` = true, `0xFF` = false (true sorts first). - **Memo (Long Text)** is **indexable** in Access (`CREATE INDEX` on a memo column succeeds — only - `OLE Object` is rejected, *"Invalid field definition … in definition of index or relationship"*). + `OLE Object` is rejected, *"Invalid field definition … in definition of index or relationship"*). ACE + refuses an OLE column on **every** route into an index: `CREATE INDEX`, a `PRIMARY KEY` or `UNIQUE` + constraint in `CREATE TABLE` or added by `ALTER TABLE`, a foreign key in either place (before its type + match is checked), and `ALTER COLUMN` of an indexed column to OLE. It refuses up front and leaves nothing + behind — no table from a refused `CREATE TABLE`, the column and its index unchanged after a refused `ALTER`. Its key is the **ordinary Text collation key over the value's first 255 characters** — verified - byte-for-byte vs ACE (`MemoKeyEncodingTests`): a 256- or 300-character memo yields exactly the key of + byte-for-byte vs ACE: a 256- or 300-character memo yields exactly the key of its 255-character prefix, so two memos differing only past character 255 share a key (fine for a non-unique index). Index keys are therefore encoded from the **logical** row values, before memo/OLE values are materialised into their `LongValueDescriptor`s. @@ -195,49 +191,43 @@ Then the value, transformed: > | soft hyphen | inline record, code `0x83` | wholly ignorable, no record | > > **v0 is the NT4-era NLS order, renumbered into one byte.** v1 could be *identified* because its primaries -> are the NLS `(SM, AW)` pair verbatim; v0's are a Jet-specific compaction, which is why its table had to be -> measured character by character instead. But the compaction turns out to be **order-preserving**, so the -> table is explained rather than merely recorded. Sorting every character by the primary in the +> are the NLS `(SM, AW)` pair verbatim; v0's are a Jet-specific compaction, so its table is measured +> character by character instead. But the compaction is **order-preserving** against the primaries of the > **Windows NT 4.0 – Server 2003** table (the generation contemporary with Jet 3.5/Access 97 and Jet 4/Access -> 2000) and checking v0's bytes come out non-decreasing gives **507 of 510** strictly-ordered pairs kept, and -> **947 of 955** NT4 ties still tying (`SortOrderProvenanceProbeTest`, needs `LIBRED_NT4_TABLE`). By block: +> 2000): v0's bytes keep nearly every NT4 strict ordering and nearly every NT4 tie. By block: > > | | agreement | > |---|---| > | Cyrillic, Greek, Hebrew, both Latin extensions, punctuation, currency, letterlike, number forms, spacing modifiers, fullwidth | **100%** — every block, every pair | -> | Latin-1 + ASCII | 51/52; the single exception is `U+0651`, whose `FF FF` is not a primary at all — see the shadda rule below | -> | Arabic | 149/151 — the only script where Jet genuinely renumbered against NLS | +> | Latin-1 + ASCII | all but one pair; the single exception is `U+0651`, whose `FF FF` is not a primary at all — see the shadda rule below | +> | Arabic | all but two pairs — the only script where Jet genuinely renumbered against NLS | > > So the `+2` stride, the gaps that became language-letter insertion slots, and the `0x79` page for > non-Latin scripts are all one decision: **compact the NT4 primary order into a byte, leaving room**. -> Jet also *narrowed* it — of the 552 characters v0 treats as ignorable, 464 (84%) are unweighted in the NT4 -> table too, but the remaining 88 are weighted by NLS and dropped by Jet, which is an editorial choice of its -> own and not something a published table would have told us. -> -> v1's table is **very nearly** the Windows Server 2008 sorting weight table, frozen — identified by -> reconstructing measured ACE v1 keys from every published Windows table (Server 2008 scores 25/25; -> Win7/2008R2 24/25, Vista 23/25, Win8+ 22/25, NT4-2003 18/25, the discriminators being `1` = `13 25` vs -> `13 26`, its DW `2` vs `3`, and `½` = `13 24 214` vs `13 17 2`). Access 2010 shipped with the then-current -> weights and froze them when Windows 7/8 moved them — the "major NLS version, re-index everything" event -> described in [MS-UCODEREF] and *Handling Sorting in Your Applications*. -> -> **"Very nearly" is load-bearing.** Those 25 discriminators were all Latin and symbols, and a full-BMP sweep -> shows the published file is not what ACE carries everywhere: it is right about 57,793 characters and wrong -> about 501, plus 5,082 that ACE treats as wholly ignorable and the published file has no entry for at all. -> The disagreements are concentrated in scripts added or reweighted after Server 2008 — ACE gives Balinese -> and Canadian syllabics *Latin* weights — and in the Arabic harakat and several ligature blocks. Rather -> than guess at which NLS revision ACE really carries, the differences are **measured and embedded** -> (`SortKeyTableV1Overrides.bin`, 2.0 KB, written by `SortKeyTableV1OverrideGeneratorTest` with -> `LIBRED_GENERATE_V1=1`) — the same answer v0 needed, at 3% of the size, because v1 is right about the rest. +> Jet also *narrowed* it — most characters v0 treats as ignorable are unweighted in the NT4 table too, but +> about one in six is weighted by NLS and dropped by Jet, an editorial choice of Jet's own. +> +> v1's table is **very nearly** the Windows Server 2008 sorting weight table, frozen. The discriminators +> against neighbouring Windows tables include `1` = `13 25` vs `13 26`, its DW `2` vs `3`, and `½` = +> `13 24 214` vs `13 17 2`. Access 2010 shipped with the then-current weights and froze them when Windows 7/8 +> moved them — the "major NLS version, re-index everything" event described in [MS-UCODEREF] and *Handling +> Sorting in Your Applications*. +> +> **"Very nearly" is load-bearing.** Across the BMP the published file is not what ACE carries everywhere: it +> is wrong about 501 characters, plus 5,082 that ACE treats as wholly ignorable and the published file has no +> entry for at all. The disagreements are concentrated in scripts added or reweighted after Server 2008 — ACE +> gives Balinese and Canadian syllabics *Latin* weights — and in the Arabic harakat and several ligature +> blocks. LibRed therefore embeds the differences as **measured overrides** (`SortKeyTableV1Overrides.bin`, +> 2.0 KB) on top of the published table. > > An override records the primary and secondary bytes **raw**, not as `(SM, AW, DW)` weights, because that > reading assumes every primary is a two-byte pair carrying one secondary and ACE breaks it both ways: the > Arabic harakat have a secondary and *no primary* (`U+064C` is `7F 01 56 00`), and the Lao vowel signs take > a **one-byte** primary (`U+0EB0` is `7F 41 01 0A 00`). A primary byte can even *be* `0x01`: `U+0385`, > `U+1B3B` and `U+FC25` weigh `07 53 01`, and `U+FC33` and `U+FCC2` weigh `29 0B 01`, so the section -> delimiter is the **last** `0x01` in a key, not the first. Splitting at the first made those five look like -> a key with an extra section bolted on; measuring them in combination (`aX`, `Xa`, `XaX`) showed they are -> ordinary two-weight expansions. +> delimiter is the **last** `0x01` in a key, not the first. Splitting at the first makes those five look like +> a key with an extra section bolted on; in combination (`aX`, `Xa`, `XaX`) they are ordinary two-weight +> expansions. > > This also explains the framing generally: **script member 6 is the word-sort class**, and the apostrophe's > `0x80` and hyphen's `0x82` inline codes are simply their Alphabetic Weights — so the inline record is @@ -251,15 +241,13 @@ Then the value, transformed: > the two weights. > > The two readings agree below `0x100` and diverge above it, and the offset `0x07 + 4 x position` passes -> `0xFF` at position 62. So a hyphen at character 63 is `81 03`, at 200 `83 27`, at 250 `83 EF` — measured -> against ACE across positions 10 to 250 under both orders. +> `0xFF` at position 62. So a hyphen at character 63 is `81 03`, at 200 `83 27`, at 250 `83 EF` — verified +> against ACE under both orders. > -> Worth stating loudly, because it is invisible to the obvious tests. Every single character encodes -> correctly, every short string encodes correctly, and the field only overflows past character 62 — so -> reading `0x80` as a marker and truncating the position looked right everywhere anyone had looked, and -> silently produced a wrong key for any longer value containing an apostrophe or hyphen. A hyphenated name in -> a 255-character column is enough. The lesson is to measure COMBINATIONS and not only characters: a -> per-character sweep can be exhaustive — all 63,422 of them — and still miss a whole class of bug. +> Reading `0x80` as a marker byte and truncating the position gives the right key for every single character +> and every short string — the field only overflows past character 62 — and a silently wrong key for any +> longer value containing an apostrophe or hyphen. A hyphenated name in a 255-character column is enough, and +> no per-character sweep can show it. > > **French is the diacritic section written BACKWARDS — no tailored letter at all.** The same pseudocode has > an `IsReverseDW` flag whose rule is: drop the run of default diacritics from the **left** rather than the @@ -272,13 +260,11 @@ Then the value, transformed: > | `côté` | `02 12 02 0E` | `12 02 0E` | `01 0E 02 12 00` | > > So French orders by the LAST accent — `cote < côte < coté < côté`, where General gives -> `cote < coté < côte < côté`. LibRed matches ACE across all of Latin-1 and Latin Extended-A with accents -> doubled and tripled per string, 1,289 values, zero differences. +> `cote < coté < côte < côté`. LibRed matches ACE across Latin-1 and Latin Extended-A with accents doubled and +> tripled per string. > -> It sat in the "unclassified, secondary-section tailoring" bucket for a long time, and the reason is worth -> keeping: a word with ONE accent encodes identically under both orders, and the sample set that measured -> every locale against General contained no two-accent word. The rule was invisible to the measurement, not -> absent from it — the same shape of blind spot as the inline position field above. +> A word with ONE accent encodes identically under French and General, so only a multi-accent word shows the +> difference — the same shape of blind spot as the inline position field above. > > **And the `01 01 01` before a word-sort record is three SECTION SEPARATORS, not an introducer.** The same > pseudocode gives the full frame as @@ -287,41 +273,37 @@ Then the value, transformed: > primaries 01 diacritics 01 case-weights 01 extra-weights 01 special-weights 00 > ``` > -> Access emits that frame with the **case-weight section empty**, which is the mechanism behind something -> long known here empirically: case and character width fold because width lives in bit 0 of the Case Weight, -> and Access simply never writes that section. So the run of three is end-of-diacritics, an empty case -> section, an empty extra section — and it shortens to `FF 01` when a kana section fills the extra slot. +> Access emits that frame with the **case-weight section empty**, which is why case and character width fold: +> width lives in bit 0 of the Case Weight, and Access never writes that section. So the run of three is +> end-of-diacritics, an empty case section, an empty extra section — and it shortens to `FF 01` when a kana section fills the extra slot. > `MIN_DW = 2` in the same source is the `0x02` default secondary whose trailing run gets trimmed. > > Three more things that source settles, or usefully fails to: > -> - **The contraction limit corroborates v0's provenance independently.** It supports only 2- and -> 3-character contractions on NT4 through Server 2003, and 4- to 8-character ones from Vista. Every v0 -> tailoring here tops out at three (Hungarian `ggy`) — arrived at by measurement, and matching the -> generation the weight-table comparison already identified. Two unrelated routes to the same date. +> - **The contraction limit corroborates v0's provenance.** It supports only 2- and 3-character +> contractions on NT4 through Server 2003, and 4- to 8-character ones from Vista. Every v0 tailoring tops +> out at three (Hungarian `ggy`), matching the generation the weight-table comparison identifies. > - **The `FD FF` Han primary is NOT the Windows 7 three-byte weight.** That feature emits `SM PW DW` — > *three* bytes, with the diacritic moved into the primary and omitted from its own section — and arrived -> in Windows 7 / Server 2008 R2, *after* the table Access froze. Ours is four bytes and is the older +> in Windows 7 / Server 2008 R2, *after* the table Access froze. ACE's is four bytes and is the older > extension-marker shape, alongside `SCRIPT_MEMBER_EXT_A` / `PRIMARY_WEIGHT_EXT_A`. Consistent with the > freeze; the measured bytes stand. > - **Access PACKS the East Asia extra weights where Windows does not.** The specification gives one byte per > character per group (`W6`, `W7`, trailing `0xE4` trimmed, `0xFF` between). Access instead packs the kana -> flags three to a byte — measured across all thirty combinations up to four kana. So the kana section is a -> compacted variant of the documented structure rather than the structure itself. +> flags three to a byte (below). So the kana section is a compacted variant of the documented structure +> rather than the structure itself. > -> **Nothing in that source covers the 510-byte cap, truncation or the checksum below.** A useful negative: -> those are Jet/ACE inventions with no Windows counterpart, which is why they had to be measured. +> **Nothing in that source covers the 510-byte cap, truncation or the checksum below.** Those are Jet/ACE +> inventions with no Windows counterpart. > > LibRed encodes both: `JetTextCollation` (v0, a measured table) and `JetTextCollationV1` (v1, the published > table plus the measured overrides — see `tools/sortkey-table/generate.ps1`), sharing `JetKanaSection`. -> **Both now cover the whole Basic Multilingual Plane**: 63,422 characters each, every key byte-for-byte -> what ACE stores, nothing refused and nothing left unhandled (`Probe_full_bmp_coverage`, needs -> `LIBRED_FULL_BMP`). Other locales are **not** refused under v1: the six orders with a version-1 table are -> tailored, and every other LANGID falls back to General v1 (see §10.4's collation list). +> **Both cover the whole Basic Multilingual Plane**: every key byte-for-byte what ACE stores, nothing refused +> and nothing left unhandled. Other locales are **not** refused under v1: the six orders with a version-1 +> table are tailored, and every other LANGID falls back to General v1 (see §10.4's collation list). > -> **Above the BMP** the two orders disagree completely, measured over all of planes 1 and 2 and sampled across -> all sixteen. **v0 ignores astral characters entirely** — every one gets the empty key `7F 01 00`, so under -> General Legacy an astral character is invisible to the index. **v1 weighs both surrogate halves**, each +> **Above the BMP** the two orders disagree completely. **v0 ignores astral characters entirely** — every one +> gets the empty key `7F 01 00`, so under General Legacy an astral character is invisible to the index. **v1 weighs both surrogate halves**, each > looked up in the table like any other character: `U+10000` is `7F B002 B4F8 01 3F 3F 00`, the high surrogate > `D800` weighing `B002` and the low `DC00` weighing `B4F8`. > @@ -330,83 +312,9 @@ Then the value, transformed: > the same. So planes 1 and 2 are fully distinguished, while planes 3 to 16 collapse onto **1,024 keys** and > any two code points there congruent mod `0x400` share one. > -> The only change v1 needed was to treat an **unweighted surrogate as ignorable rather than an error**. The -> tempting reading of the plane-3 samples — "the high surrogate contributes nothing" — is wrong, and skipping -> every high surrogate breaks all 131,068 characters of planes 1 and 2. `AstralCollationProbeTest`, needs -> `LIBRED_ASTRAL=1` (or `LIBRED_ASTRAL_FULL=1` for a whole plane). - -### 10.5 The 510-byte index entry limit - -**ACE stores an index entry of at most 510 bytes as built.** At exactly 510 it comes back byte-for-byte; a -value that would need 511 comes back as 510: the first **508** bytes kept, and the rest replaced by a -two-byte **checksum over the bytes that were dropped**. That is why two long values sharing a 508-byte -prefix still sort apart instead of colliding. - -#### The checksum - -A 16-bit fold over the **discarded run** — every byte from offset 508 to the end of the untruncated key — -stored big-endian in the last two bytes. Each byte is XORed into the **high** half and the state is then -folded; the fold happens *between* bytes, so the last byte of the run contributes its XOR and nothing else: - -``` -crc = 0 -for each byte b of the discarded run: - crc ^= b << 8 - if b is not the last: crc = (crc >> 8) ^ T[crc & 0xFF] -``` - -with no initial value and no final XOR. The step table is - -``` -T[1<> 8) ^ T[(crc ^ b) & 0xFF]`, passing the byte **through** the table. Sweeping all 65,536 -polynomials in five conventional framings found nothing, which is what pointed at the framing. - -The table is not a guess either. The function is **affine over GF(2)** — three tails differing in one byte -give `L(0xA3) = CA03`, `L(0x13) = 6980`, `L(0xB0) = A383`, and `CA03 ^ 6980 = A383` exactly — and -**shift-invariant** across 173 observations, so a byte at distance *d* from the end contributes `S^(d-1)` of -itself whatever the message length. That makes the eight rows above solvable by Gaussian elimination over the -measured contributions, and the solution predicts all 657 of them. - -Equivalently, and how `JetIndexKeyChecksum` implements it: fold every byte but the last in the form -`crc = (crc >> 8) ^ T[crc & 0xFF] ^ b`, then XOR the last byte's `b << 8` into the result. The two are the -same function — verified over 12,800 random inputs at every length from 1 to 64 bytes. - -**The discarded run is 3 bytes at minimum** (truncation triggers only above 510, and the run is -`length − 508`), and the rule is keyed to the run's *last byte*, not to a fixed offset in the key: measured -over runs of 3 through 13 bytes, from keys of 511 to 521 bytes. - -Verified against ACE for Latin, accented and Han text under both sort orders; for composite keys ending in -`LONG`, `CURRENCY` and `DOUBLE`; and for keys whose dropped bytes contain an inline **word-sort record** — -ACE does not reposition that record when truncating, so it reconstructs exactly (measured at eight positions -for each of the hyphen and apostrophe). **Nothing is refused**; every key past the cap is truncated the way -ACE truncates it. - -> The "fold between bytes" framing matters, and is easy to get wrong in a way no all-text test can catch. -> Writing the loop as "fold every byte except the last" — reading the skipped byte as the text terminator — -> gives the identical answer whenever that byte is `0x00`, which it always is when the key ends in text. The -> two readings part company the moment the last key column is numeric. See -> [`docs/design/index-key-checksum.md`](../design/index-key-checksum.md) for how that was found and what it -> cost. - -The cap is on the **whole entry, not per column**: two 200-character text columns weigh about 404 bytes of -key each, comfortably under the cap individually, and ACE stores their combined entry hashed at 510. - -Because it limits **weights** rather than characters, the text it buys depends on collation and script — and -this is the practical cost of General over General Legacy, invisible in the schema: - -| | bytes per character | characters indexed in full | -|---|---|---| -| v0, Latin | 1 primary | **255** — the column limit is reached first | -| v0, accented / CJK | 2 | **254** | -| v1, Latin | 2 primary | **253** | -| v1, accented | 3 | **169** | -| v1, Han | 4 (`FD FF AW DW`) | **127** | Tests: `GeneralV1CollationTests` (keys -> measured from ACE) and `GeneralV1CollationAccessTests` (live oracle, plus ACE seeking an index LibRed -> wrote in a v1 database). +> So a v1 encoder treats an **unweighted surrogate as ignorable rather than an error**. The tempting reading +> of planes 3 and up — "the high surrogate contributes nothing" — is wrong: skipping every high surrogate +> breaks every character of planes 1 and 2. - **Text:** Jet's "General" collation. The key is the start flag, then one or two **primary-weight** bytes per character, then a `01 00` terminator. Weights are **case-folded** @@ -414,35 +322,33 @@ this is the practical cost of General over General Legacy, invisible in the sche space weighs `0x07`. Most characters weigh one byte; `^ _ \` { | } ~` weigh two (sharing the `0x2B` page). The weight table is a fixed lookup (A=`4A`, B=`4C`, C=`4D`, …, digits step by two from `0x36`), **verified byte-for-byte against the ACE engine** over printable ASCII and - implemented by `JetTextCollation` — so LibRed can now *write* ASCII text index keys (e.g. a + implemented by `JetTextCollation` — so LibRed can *write* ASCII text index keys (e.g. a string primary key). Decoding remains lossy (case is discarded — that is why a text primary key treats `'A'` and `'a'` as duplicates). **Twenty characters are "ignorable"** (so `O'Brien` sorts next to `OBrien`): they add **no primary weight**, but each appends an inline record to a trailing section. - > Sixty across the BMP — twenty hand-verified below, and forty more measured into the resource (CJK and - > fullwidth punctuation, further dashes and quotation forms). + > Sixty across the BMP — the twenty below, and forty more in the embedded table (CJK and fullwidth + > punctuation, further dashes and quotation forms). > - > The hand-verified set and their codes, measured alone and inside a word so the position arithmetic is - > confirmed rather than assumed: apostrophe `0x80`, hyphen `0x82`, soft hyphen `0x83`, `U+2010` `0x84`, `U+2011` - > `0x85`, `U+2027` `0x86`, `U+2043` `0x87`, `U+2012` `0x88`, `U+2013` `0x89`, `U+2014` `0x8B`, `U+2015` - > `0x8C`, and the Arabic harakat `U+064B`–`U+0650` and `U+0652` running `0xA0`–`0xA6`. **The fullwidth - > apostrophe and hyphen share their ASCII counterparts' codes exactly** (`U+FF07` = `0x80`, `U+FF0D` = - > `0x82`) — the one place fullwidth really does collapse onto ASCII, unlike the letters. `0x8A` is unused - > by anything in the swept range. After the primary's - `0x01` end marker, if any ignorable char is present the key adds `01 01 01` once, then per - ignorable char four bytes `80 06 `, then the final `00`. ` = 0x07 + 4 × (count + > The twenty and their codes, verified alone and inside a word: apostrophe `0x80`, hyphen `0x82`, soft + > hyphen `0x83`, `U+2010` `0x84`, `U+2011` `0x85`, `U+2027` `0x86`, `U+2043` `0x87`, `U+2012` `0x88`, + > `U+2013` `0x89`, `U+2014` `0x8B`, `U+2015` `0x8C`, and the Arabic harakat `U+064B`–`U+0650` and `U+0652` + > running `0xA0`–`0xA6`. **The fullwidth apostrophe and hyphen share their ASCII counterparts' codes + > exactly** (`U+FF07` = `0x80`, `U+FF0D` = `0x82`) — the one place fullwidth really does collapse onto + > ASCII, unlike the letters. `0x8A` is unused by any measured character. + + After the primary's `0x01` end marker, if any ignorable char is present the key adds `01 01 01` once, + then per ignorable char four bytes `80 06 `, then the final `00`. ` = 0x07 + 4 × (count of **primary weights** emitted before it)` and `` is `0x80` for apostrophe / `0x82` for hyphen — verified against ACE (e.g. `ANNE-MARIE` → `… 80 17 06 82 …`, the hyphen at position 4; `Aß-B` → `7F 4A 6B 6B 4C 01 01 01 01 80 13 06 82 00`, hyphen at position **3** because ß expands to two weights `S`+`S`). - > **Weights, not bytes — an earlier revision said bytes and LibRed implemented that.** The two agree for - > everything Latin, which is why it stood so long. A two-byte weight settles it: `£-` puts the hyphen at - > `0x0B` (`0x07 + 4×1`) although `£` is `34 A7`, and `©`, `½`, `Ω`, `б` all behave the same, while `£A-` - > is `0x0F`. So both the secondary section and this one index by weight. Guarded by the `£-`/`Ω'A` family - > in `LocaleCollationAccessTests` — the older samples were Latin-only and could not see it. + > **Weights, not bytes.** The two agree for everything Latin, so only a two-byte weight tells them apart: + > `£-` puts the hyphen at `0x0B` (`0x07 + 4×1`) although `£` is `34 A7`, and `©`, `½`, `Ω`, `б` all behave + > the same, while `£A-` is `0x0F`. So both the secondary section and this one index by weight. > **Why those two characters specifically:** this is Windows' documented **word sort**, the default for > the NLS sorting functions — *"all punctuation marks and other nonalphanumeric characters, except for the @@ -468,15 +374,12 @@ this is the practical cost of General over General Legacy, invisible in the sche | `U+0651 U+0645` | `7F FFFF 79C6 01 08 00` | nothing precedes: `FF FF`, and **no secondary slot** — one secondary for two primaries | So `FF FF` is what a shadda weighs only when there is nothing to double, which is the one form a - per-character sweep can present it in — and the reason the table recorded `FF FF` as its primary and the - provenance check above counted it an anomaly. Same shape as the kana prolonged sound mark: a mark whose - weight is a function of its neighbour cannot be tabulated per character, and a single-character sweep - cannot discover it. Measured across ten shadda shapes against ACE - (`CollationSurveyProbeTests` batch 06, whose self-check is the assertion). + per-character table can hold for it — and why it appears as an anomaly against NLS above. Same shape as + the kana prolonged sound mark: a mark whose weight is a function of its neighbour cannot be tabulated per + character. Verified against ACE. **Latin-1 punctuation and symbols** weigh two bytes, in groups that mirror the Win32 NLS primary order - in ACE's own compacted numbering — harvested from ACE's stored keys character by character - (`Latin1SymbolCollationAccessTests`): + in ACE's own compacted numbering — measured from ACE's stored keys character by character: `¡ ¦ ¨ ¯ ´ ¸ ¿` = `2B 10`…`2B 16` (continuing the `^_\`{|}~` group); `± « » × ÷` = `33 04/05/07/09/0A`; `¢ £ ¤ ¥ § © ¬ ® ° µ ¶ ·` = `34 A6`…`34 B1`; `¼ ½ ¾` = `37 12/16/1A`. The **ordinal indicators** `ª`/`º` are not symbols at all: they take their base @@ -492,11 +395,10 @@ this is the practical cost of General over General Legacy, invisible in the sche > weight. So full-width forms, half-width katakana, and case all collapse for free. ACE does **not** > pre-map with `LCMAP_HALFWIDTH`: `U+3000` (ideographic space) keeps its own key `7F 07 01 00` rather than > becoming a space and being dropped by the trailing-space trim, which is what a width pre-mapping would - > produce. Ligatures need no special handling either — NLS itself expands `fi` to `f` + `i`. (Probed in - > `SortKeyComparisonProbeTest`.) + > produce. Ligatures need no special handling either — NLS itself expands `fi` to `f` + `i`. **The long s `ſ` (U+017F) is a letter of its own**, not a fold onto `s`: it takes the two-byte primary - `6C 06` — the S–T gap — in **every** v0 order measured, General included (`LocaleCollationAccessTests`). + `6C 06` — the S–T gap — in **every** v0 order measured, General included. Uppercasing it invariantly gives `S`, so it has to be matched on the original character or the distinction is lost. @@ -510,20 +412,17 @@ this is the practical cost of General over General Legacy, invisible in the sche > The weights are in an embedded resource (`SortKeyTableV0.bin`, 74 KB): 63,105 of them, 19,186 ignorable, > plus 40 word-sort ignorables and 276 kana — far past anything hand-maintainable. Most of v1's table can > be embedded from a published Microsoft file; v0's cannot at all, since its primaries are a Jet compaction - > rather than the NLS weights, so **ACE itself is the source**: `SortKeyTableV0GeneratorTest` inserts every - > code point into an indexed text column, reads the stored keys back and writes the resource - > (`LIBRED_GENERATE_V0=1`). Non-Latin scripts nearly all live on the same **two-byte `0x79` page** the - > locale tailorings use for letters sorting after Z. + > rather than the NLS weights, so **ACE itself is the source**: the table is ACE's stored key for every code + > point in an indexed text column. Non-Latin scripts nearly all live on the same **two-byte `0x79` page** + > the locale tailorings use for letters sorting after Z. > - > Both generators must run with the resource they are about to replace **suppressed** - > (`JetTextCollationV1Overrides.Suppressed`), and v1's shows why plainly: it records where the encoder - > *disagrees* with ACE, so measuring an encoder that already consults it would find no disagreements and - > write an empty file. Suppressing from the outset also means a generator never has to be able to *read* - > the resource it replaces, so it bootstraps from a stale or absent one. + > Regenerating either table must run with the resource being replaced **suppressed** + > (`JetTextCollationV1Overrides.Suppressed`): v1's records where the encoder *disagrees* with ACE, so an + > encoder that already consults it finds no disagreements and writes an empty file. > - > Two things that only a full sweep would show. **ACE weighs every CJK ideograph and the entire private-use - > area** — `U+5000`–`8FFF`, `B000`–`CFFF` and `E000`–`EFFF` are 4,096 for 4,096, none of it ignorable. And - > across all 65,536 code points **ACE refused exactly one**. + > **ACE weighs every CJK ideograph and the entire private-use area** — `U+5000`–`8FFF`, `B000`–`CFFF` and + > `E000`–`EFFF` are 4,096 for 4,096, none of it ignorable. And across all 65,536 code points **ACE refuses + > exactly one**. > > **Kana are their own mechanism.** A kana takes the two-byte primary `7F `, and the key gains a > section of its own — so a single kana changes the shape of the whole key: @@ -542,7 +441,7 @@ this is the practical cost of General over General Legacy, invisible in the sche > **The small/normal flags are bit-packed.** Trailing normal forms are dropped, then what remains goes > three per byte, two bits each, most significant first, under a `10` marker in the byte's top two bits — > `11` normal, `10` small, `00` padding. One small kana is `A0`, "normal small" is `B8`, four kana take two - > bytes with the marker repeated. Verified over all 30 combinations up to four kana. + > bytes with the marker repeated. Verified for every combination up to four kana. > > Two rules that only appear in multi-character strings. The **halfwidth voicing marks `U+FF9E`/`U+FF9F` > are combining** — measured alone they look ignorable, but ACE folds them into the preceding kana's @@ -580,7 +479,7 @@ this is the practical cost of General over General Legacy, invisible in the sche > > What `02 80 FF 80` denotes is still not established; it never varies, so it is emitted as a literal. > - > Three categories emerged that the Latin-1 range never showed: + > Three categories that the Latin-1 range does not show: > - **Ignorable** — ACE stores *nothing at all* (key `7F 01 00`): no primary, not even a secondary slot. > Romanian's comma-below `ș`/`ț` are in this class, which is why they appear to "keep General's weights": > General has none for them. Ignorability held in every order measured. @@ -588,16 +487,15 @@ this is the practical cost of General over General Legacy, invisible in the sche > Hebrew's niqqud, the Cyrillic combining marks and three Greek ones work this way. > - **Locale-dependent expansion** — `DŽ` is `D`+`Ž` (two weights, the caron on the second: `7F 4F 78 01 02 > 14 00`), and `Ǣ` is `Æ` with a macron whose letters differ per locale — Icelandic gives it its own `Æ` - > at `79 04`. These are **refused** rather than approximated, since one weight where ACE uses two is - > silently wrong in any string with a later accent. + > at `79 04`. One weight where ACE uses two is silently wrong in any string with a later accent, so these + > are encoded by decomposition (below), never approximated. > - > **Locales share the block tables**, because measuring all 21 against General showed the departures are - > tiny: **27 entries in total across every locale**, and most add only one or two over the entire extended - > range (Croatian eleven, the outlier). Each is listed in that locale's tailoring, which is consulted - > first — Lithuanian retailors fullwidth `Ｙ`, Ukrainian moves `ь`, Swedish puts wynn on `v` because it - > makes `w` a variant of `v`. Estonian, by contrast, leaves fullwidth `Ｖ` on General's weight, so these - > really are per-locale facts rather than a rule. `LocaleCollationAccessTests` asserts the whole range for - > every order, so a missed departure fails the build rather than writing a silently wrong key. + > **Locales share the block tables**, because the departures from General are tiny: **27 entries in total + > across every locale**, and most add only one or two over the entire extended range (Croatian eleven, the + > outlier). Each is listed in that locale's tailoring, which is consulted first — Lithuanian retailors + > fullwidth `Ｙ`, Ukrainian moves `ь`, Swedish puts wynn on `v` because it makes `w` a variant of `v`. + > Estonian, by contrast, leaves fullwidth `Ｖ` on General's weight, so these really are per-locale facts + > rather than a rule. > > **A ligature character weighs as its decomposition**, one component at a time — there is no ligature > mechanism in the format at all. `DŽ` encodes exactly as the string `DŽ` (`7F 4F 78 01 02 14 00`), `LJ` as @@ -611,11 +509,11 @@ this is the practical cost of General over General Legacy, invisible in the sche > do not decompose: Icelandic's `Ǣ` is its own `Æ` plus a secondary, and Croatian's `DŽ` is its single-letter > `dž`. The components do take the locale's letters, though — Slovenian's `DŽ` is `D` plus *Slovenian's* `ž`. > - > **Coverage is complete: all 2,147 characters ACE encodes, for every one of the 23 orders, with zero - > mismatches.** Nothing in the swept range is refused and nothing disagrees with ACE. + > **Coverage is complete over that range: every character ACE encodes, for every order listed below.** + > Nothing in it is refused and nothing disagrees with ACE. **Diacritic secondary weights**, each depending only on the mark and not the base letter — derived from - ACE by `TailoringGeneratorProbeTest`: acute `0x0E`, grave `0x0F`, **dot above `0x10`**, circumflex `0x12`, + ACE: acute `0x0E`, grave `0x0F`, **dot above `0x10`**, circumflex `0x12`, diaeresis `0x13`, **caron `0x14`**, **breve `0x15`**, **macron `0x17`**, tilde `0x19`, ring `0x1A`, **ogonek `0x1B`**, cedilla `0x1C`, **double acute `0x1D`**. Atomic letters that do not decompose carry one directly: `Ø`→`O`+`0x21`, `Ð`→`D`+`0x68`, **stroke** `Đ`→`D`+`0x1E`, `Ħ`→`H`+`0x1E`, `Ł`→`L`+`0x1F`, @@ -632,10 +530,9 @@ this is the practical cost of General over General Legacy, invisible in the sche > **The secondary section has one entry per primary *weight*, not per primary *byte*.** A weight may be > one byte or two, and a two-byte weight still takes a single slot — Norwegian `ö` is - > `7F 79 06 01 13 00`: two primary bytes, one secondary. This only becomes visible once two-byte primaries - > and accents appear together, which is why it surfaced with the locale tailorings - > (`Ångström` in Norwegian, where `å` and `ö` are both two-byte). The **inline** apostrophe/hyphen section - > below counts weights as well, so both sections index the same way. An expansion is several *weights* + > `7F 79 06 01 13 00`: two primary bytes, one secondary. This is visible only where two-byte primaries and + > accents appear together (`Ångström` in Norwegian, where `å` and `ö` are both two-byte). The **inline** + > apostrophe/hyphen section below counts weights as well, so both sections index the same way. An expansion is several *weights* > (`ß`→`SS` is two one-byte weights), so it takes two slots. The section is emitted only when some character is accented: after the primary's `0x01` end marker it @@ -644,7 +541,7 @@ this is the practical cost of General over General Legacy, invisible in the sche `7F 60 51 75 59 4D 64 07 4F 1C 53 1C 01 02 0E 00` (é = primary `0x51` = E, secondary `0x0E`), and `Montréal` (é at position 5) → `… 01 02 02 02 02 02 0E 00`. LibRed decomposes via Unicode NFD (base letter + combining mark) plus the small atomic table above; `JetTextCollation` reproduces these keys - **byte-for-byte vs ACE** (México/Montréal/München/São Paulo/Résumé and single accents). + **byte-for-byte vs ACE**. **Descending** text keys are the **bitwise inverse of the ascending key, with a `0x00` appended** — verified against ACE (e.g. ascending `A` = `7F 4A 01 00` → descending @@ -652,30 +549,28 @@ this is the practical cost of General over General Legacy, invisible in the sche the fixed-type keys. **Locale-specific orders.** A database can be created with a sort order other than General; Access exposes - them as the "New Database Sort Order" list. Verified against **29 Access-authored fixtures — every non-CJK - entry in that list**, in `Data/`, each diffed against General v0 by having ACE encode the same 193 samples - and reading the stored keys back (`LocaleFixtureCollationProbeTest`, plus `DaoLocaleCollationProbeTest` for - orders only DAO can name): + them as the "New Database Sort Order" list. Verified against ACE for **every non-CJK entry in that list**, + each compared against General v0 (plus orders only DAO can name): - The stored value is a **true LCID**, not a small enum — Spanish Traditional is `1034` (`0x040A`) and Spanish **Modern** is `3082` (`0x0C0A`). DAO's `CollatingOrderEnum` lists only `dbSortSpanish = 1034`; - the Modern order postdates it and has no DAO name. Both files are **sort-order version `0`**, so the + the Modern order postdates it and has no DAO name. Both are **sort-order version `0`**, so the version is **orthogonal to the locale** — though in practice few locales have both generations. Access's "New Database Sort Order" list names a legacy order separately (`General - Legacy`, `Romanian - Legacy`, `Croatian - Legacy`, `Japanese - Legacy`), and **neither Spanish order has a `- Legacy` twin**: a second generation exists only where the Windows tailoring actually changed. - - **Version 1 is not a General-only thing.** Five of the fixtures stamp version `1` — Bosnian, Croatian, + - **Version 1 is not a General-only thing.** Five orders in the list stamp version `1` — Bosnian, Croatian, Indic, Romanian, Serbian — and all encode with **2-byte NLS primaries** exactly as General v1 does - (`a` = `7F 0E 02 01 00`, `c` = `7F 0E 0A 01 00`, `d` = `7F 0E 1A 01 00`). Every one of 193 samples differs - from General v0, because the whole key shape changes rather than individual letters moving. Croatian and + (`a` = `7F 0E 02 01 00`, `c` = `7F 0E 0A 01 00`, `d` = `7F 0E 1A 01 00`). Every key differs from + General v0, because the whole key shape changes rather than individual letters moving. Croatian and Romanian are the ones Access offers in both generations, and their `- Legacy` twins are ordinary v0 files with the same LANGID; Bosnian, Indic and Serbian have no legacy twin at all. - **The whole four-byte field is one 32-bit LCID** (§3.4). Several entries in Access's list are Windows *alternate sort orders*, which live in the LCID's high word and share their LANGID with the base locale: - | fixture | raw `0x6E`..`0x71` | LANGID | sort id | version | LCID | + | order | raw `0x6E`..`0x71` | LANGID | sort id | version | LCID | |---|---|---|---|---|---| | `German Phone Book` | `07 04 01 00` | 1031 | `01` | 0 | `0x00010407` | | `Hungarian Technical` | `0E 04 01 00` | 1038 | `01` | 0 | `0x0001040E` | @@ -684,7 +579,7 @@ this is the practical cost of General over General Legacy, invisible in the sche | `Croatian` / `Croatian - Legacy` | `1A 04 00 01` / `1A 04 00 00` | 1050 | `00` | 1 / 0 | `0x0000041A` | Hungarian and Hungarian Technical differ **only** in the sort id, so an implementation that reads the - LANGID alone cannot tell them apart — LibRed could not, until these fixtures. + LANGID alone cannot tell them apart. - **German Phone Book is an expansion, not an insertion**: `ä` = `7F 4A 51 01 00`, i.e. primaries `a` + `e` (General has `7F 4A 01 13 00`, `a` + umlaut secondary); likewise `ö` → `o`+`e` and `ü` → `u`+`e`. It uses @@ -711,10 +606,9 @@ this is the practical cost of General over General Legacy, invisible in the sche `ą` `06` — and three in `0x4E` (`ç` `02`, `ć` `03`, `č` `04`). Estonian's `0x6C` holds `š` `06`, `z` `07`, `ž` `08`, in exactly Estonian alphabet order. Swedish/Finnish and Norwegian/Danish both stack their three extra vowels after Z: `å` `05` / `ä` `07` / `ö` `08` for Swedish, `æ` `04` / `ø` `06` / `å` `09` for - Norwegian — each language's own order. (An earlier revision here said cross-locale disagreement ruled a - sub-position out. It does not: it only ruled out a *fixed marker*. The values are per-locale ordinals.) - How a specific value is chosen is still unknown — they are ordered but not dense, and Latvian uses `0x12` - for `ķ` and `0x0C` for `ņ`. + Norwegian — each language's own order. The second byte is a per-locale ordinal, not a fixed marker, which + is why locales disagree on it. How a specific value is chosen is still unknown — they are ordered but not + dense, and Latvian uses `0x12` for `ķ` and `0x0C` for `ņ`. - **The after-Z letters use the `0x79` page**, which is where General already keeps Greek and Cyrillic: Czech `ž` = `79 05`, Polish `ż` = `79 04`, Icelandic `þ` `03` / `æ` `04` / `ö` `05`. Same two-byte @@ -755,11 +649,11 @@ this is the practical cost of General over General Legacy, invisible in the sche - **Every order is General plus a small tailoring** — including the version-1 ones. Compared against the General order of **its own version** (the v1 baseline is a database LibRed creates with - `Collation.General`, which ACE then encodes into), no order departs in more than 47 of 193 samples, and - a version-1 order is *not* a wholesale reweighting — it only looked like one against a v0 baseline, - because the key shape changes. `LocaleFixtureCollationProbeTest` reports both. + `Collation.General`, which ACE then encodes into), no order departs in more than 47 of 193 sample + values, and a version-1 order is *not* a wholesale reweighting — it only looks like one against a v0 + baseline, because the key shape changes. - | departure | orders | + | departure (of 193) | orders | |---|---| | 47 | Hungarian Technical | | 14–16 | Bosnian, Croatian, Croatian Legacy, Estonian, Serbian (16), Slovak (15), Czech, Hungarian (14) | @@ -770,24 +664,21 @@ this is the practical cost of General over General Legacy, invisible in the sche Croatian, Bosnian and Serbian depart in the same 16 as Croatian Legacy — the same letter set tailored in both generations, so a locale's *character list* is version-independent even though its weights are not. - Two caveats on reading this as effort. 193 samples are a sample, not an alphabet: `0 differ` means - *indistinguishable over these*, and a real implementation needs a fuller sweep per locale. And **French - is under-measured** — its one difference is in the *secondary section*, consistent with French ordering - accents from the end of the word, which single-character samples cannot exercise. + A sample is not an alphabet: `0` means *indistinguishable over these values*, not identical. French's + single departure is in the *secondary section*, consistent with its reversed diacritic section (above). - **Some orders are recorded but unimplemented — including one Access itself lists.** `Arabic` (1025), `Greek` (1032), `Hebrew` (1037), `Dutch` (1043) and `Cyrillic` (1049) are created happily by DAO, land on page 0 with the right LCID, get stamped onto the columns ACE itself creates, and ACE opens and runs DDL - against them — yet the keys are **byte-identical to General across 57 samples**, chosen to include what a - tailoring would actually move (Greek tonos and final sigma, Cyrillic `ё`/`й`/`ь`/`ъ`, Hebrew final forms, - Arabic hamza forms, the `ij` ligature). Access's list offers none of those five. But `Georgian Modern` - **is** in the list, carries sort id `0x01`, and is likewise indistinguishable from General over 193 - samples — so appearing in the UI does not imply an implementation, and the sort id can be recorded for an - order that does nothing. - - - **DAO can author a locale order**, even though it cannot author a sort-order *version* - (`DaoDatabaseCreationProbeTest`). A DAO-created `LANGID=0x040A` database reproduces the Access-authored - `SpanishTraditional.accdb` keys byte-for-byte, so locale fixtures need no manual Access step. + against them — yet the keys are **byte-identical to General**, including for what a tailoring would + actually move (Greek tonos and final sigma, Cyrillic `ё`/`й`/`ь`/`ъ`, Hebrew final forms, Arabic hamza + forms, the `ij` ligature). Access's list offers none of those five. But `Georgian Modern` **is** in the + list, carries sort id `0x01`, and is likewise indistinguishable from General — so appearing in the UI does + not imply an implementation, and the sort id can be recorded for an order that does nothing. + + - **DAO can author a locale order**, even though it cannot author a sort-order *version*: a DAO-created + `LANGID=0x040A` database produces the same keys, byte-for-byte, as an Access-authored Spanish Traditional + one. - `ñ` is a **letter in both Spanish orders** and an accented `n` in General — so **Modern = General plus that one letter**, and **Traditional = Modern plus the two digraphs**. Every other sample encodes byte-identically across all three orders. @@ -795,11 +686,11 @@ this is the practical cost of General over General Legacy, invisible in the sche expansions above. `chico` is `7F 4E 04 59 4D 64 01 00` — five characters, four primaries. Case folds as usual, so `ch`, `Ch` and `CH` share a key. - **LibRed implements the tailorings whose every difference is a single character** — `JetLocaleTailoring`, - a per-locale `char` → primaries override consulted ahead of the General tables, looked up by the *original* - character before the uppercased one (which is what lets Turkish disagree with invariant casing, where `I` - is the dotless letter). Implemented and asserted byte-for-byte against ACE over 345 values — the whole of - printable ASCII, Latin-1 and Latin Extended-A, plus words (`LocaleCollationAccessTests`): + **LibRed implements the tailorings below** — `JetLocaleTailoring`, a per-locale override from a character + or a character sequence (digraph, contraction) to primaries, matched longest-first ahead of the General + tables and looked up by the *original* character before the uppercased one (which is what lets Turkish disagree with invariant casing, where `I` + is the dotless letter). Verified byte-for-byte against ACE over printable ASCII, Latin-1 and Latin + Extended-A, plus words: | order | tailoring | |---|---| @@ -829,11 +720,10 @@ this is the practical cost of General over General Legacy, invisible in the sche An **empty** tailoring is meaningful and different from none: it says the order was measured to need no change, so the order can be encoded rather than refused. - **One order can wear several LCIDs.** 49 further orders needed no tailoring of their own, because their - keys are byte-identical to one already listed — measured LibRed-against-ACE over the whole survey sample - set, so the encoder was already emitting the right bytes and only the dictionary key was missing. They live - in `JetLocaleTailoring.Aliases`, which points each at the entry already built **by reference**, so the - weights exist once and two LCIDs cannot drift apart: + **One order can wear several LCIDs.** Further orders need no tailoring of their own, because their keys are + byte-identical, verified against ACE, to one already listed. They live in `JetLocaleTailoring.Aliases`, + which points each at the entry already built **by reference**, so the weights exist once and two LCIDs + cannot drift apart: | is | LCIDs | |---|---| @@ -845,19 +735,18 @@ this is the practical cost of General over General Legacy, invisible in the sche | its own language's order | the neutral `cs` `hr` `hu` `is` `pl` `ro` `sk` `sl` `et` `lv` `lt` `vi` `th` `tr` `uk` `mk` | > **Mexico is the warning against reasoning from names.** It takes Spanish *Traditional* while seventeen of - > its siblings take *Modern* and three take neither; the sweep ranks Modern as the nearest miss at 14 - > departures, so the sample set genuinely discriminated rather than picking between two plausible parents. + > its siblings take *Modern* and three take neither; Modern is 14 departures away from it, not a near tie. > **Danish 1030 vs Norwegian 1044 is not a typo.** DAO's own `dbSortNorwDan` constant is 1030, while the > order Access calls "Norwegian/Danish" — and which LibRed's `Norwegian` member names — is 1044. The two - > LCIDs carry the same order; the sweep says so directly rather than by reasoning from the names. + > LCIDs carry the same order, measured rather than inferred from the names. > **"Technical" is not a variant of the digraph order.** Hungarian Technical tailors plain `g` to `56 03`, > so its `gy` is that tailored `g` followed by an ordinary `y` — not a contraction. It is the largest > single-character tailoring measured and contains no multi-character entry. - > **A single-character sweep cannot find a digraph.** Vietnamese looked like a single-character order until - > `Ångström` came out three weights short: `ng` and `tr` each weigh as one letter. Its set is + > **A single-character sweep cannot find a digraph.** Vietnamese looks like a single-character order until a + > word contains one: `ng` and `tr` each weigh as one letter (`Ångström` is three weights short). Its set is > `ch gi kh ng nh ph qu th tr` — and note `gh` and `ngh` are *not* letters, they fall out of greedy > matching as `g`+`h` and `ng`+`h`, which is exactly what ACE stores. @@ -867,24 +756,22 @@ this is the practical cost of General over General Legacy, invisible in the sche with ACE about where rows sort. Widening it is therefore always a matter of adding a measurement, never of adding a fallback. - **The 325 orders with no tailoring at all.** Every other locale DAO will create — Bulgarian, Russian, + **The orders with no tailoring at all.** Every other locale DAO will create — Bulgarian, Russian, Greek, Hebrew, Arabic, the Indic scripts, Khmer, Yi, all fourteen Arabic and thirteen English - sublanguages, all eight Sami, and 280 more — was measured to produce index keys **byte-identical to - General v0**, over its own script's whole Unicode block plus a 626-value Latin baseline. They carry no + sublanguages, all eight Sami, and the rest — produces index keys **byte-identical to General v0**, + verified over its own script's whole Unicode block plus a Latin baseline. They carry no weights and are admitted by `JetLocaleTailoring.GeneralV0`, a set of orders rather than a dictionary of tailorings, since the fact recorded is about the order and not about anything it does. - > **The reachable set is far wider than Access's dropdown, in two directions nobody expects.** DAO takes a - > raw LANGID, so the sweep is over every LANGID Windows defines a culture for — 409 of them, of which 406 - > create. + > **The reachable set is far wider than Access's dropdown, in two unexpected directions.** DAO takes a raw + > LANGID, and all but three of the LANGIDs Windows defines a culture for create. > > **Neutral LANGIDs are real orders.** `0x0001`–`0x0091`, and the script-neutral `0x64xx`–`0x7Cxx` forms, > are all accepted and stored verbatim as collating orders `1`–`145` and `25626`–`31847` — numbers no > locale picker will ever show. Each resolves to *its language's own order*, not to General: `cs` (5) is > Czech, `hr` (26) is Croatian, `th` (30) is Thai. The opposite of what "neutral" suggests. > - > **Sublanguages do not inherit.** This is the finding that makes the sweep worth its cost. **Spanish - > splits three ways**: `es` and es-MX take Traditional, seventeen Latin-American locales take Modern, and + > **Sublanguages do not inherit.** **Spanish splits three ways**: `es` and es-MX take Traditional, seventeen Latin-American locales take Modern, and > es-US, es-419 and es-CU take no tailoring at all. **French splits two ways**: fr-CH, fr-LU and fr-MC take > the French order, while fr-CD, fr-SN, fr-CI, fr-ML, fr-MA, fr-HT, fr-CM, fr-RE and fr-029 are plain > General. Same language, same script, opposite answers — and no rule derivable from the names predicts it. @@ -896,35 +783,26 @@ this is the practical cost of General over General Legacy, invisible in the sche > Georgian is the live counterexample on the second axis — `1079` at sort id 0 is in the set, `1079` at > sort id 1 is Georgian Modern with a tailoring of its own. - What remains: **Irish 1084**, the one order the survey could not measure at all — Jet accepts it, but not - in a process that has loaded the ACE OLE DB provider, so no ACE keys could be obtained to compare; and the - **CJK** orders, deliberately out of scope. Three orders — Serbian Latin 2074, Bosnian Latin 5146 and Hindi - 1081 — are unreachable at version 0 by construction: Jet refuses them with *"Incorrect collating + What remains: **Irish 1084**, unmeasured — Jet accepts it, but not in a process that has loaded the ACE + OLE DB provider, so its ACE keys are unknown; and the **CJK** orders, deliberately out of scope. Three + orders — Serbian Latin 2074, Bosnian Latin 5146 and Hindi 1081 — are unreachable at version 0 by construction: Jet refuses them with *"Incorrect collating sequence."*, and they are exactly the three already implemented at version 1. - > **Everything above is checked by creation, not just by the survey.** `CreatedDatabaseCollationAccessTests` - > enumerates `CollatingOrder` and takes every combination `IsIndexKeyEncodable` accepts — **405** of them - > (399 at version 0 plus the six orders with a version-1 table) — - > has LibRed synthesise a database in that order, then has ACE open it, build an index and write keys, and - > requires the two engines' keys to match byte for byte. So a wrong LCID in the set does not pass quietly: - > ACE would index with whatever order that LCID really names, and the keys would part company. - - *Not yet handled:* **Irish 1084**, the one order the survey could not measure, and the **CJK** orders, - deliberately out of scope. **Six** version-1 collations are implemented — General v1, Indic v1, Romanian v1, - and Croatian / Bosnian / Serbian v1 sharing one table — and the version-1 surface is now swept: those six - are the only non-CJK orders that differ from General v1, so the rest are covered by falling back to it. - Measuring v1 needed an authoring route other than DAO, which writes version 0 for every LANGID it accepts. - - > This paragraph said "characters outside ASCII + the accented Latin-1 set; every locale other than - > General" for a long time after both had been done — it was written when they were true and never - > revisited, while the sections above it grew to a whole-BMP table and hundreds of locales. A "not yet" - > list is the first thing in a spec to rot, and the only defence is to treat it as a claim needing the - > same evidence as any other. + > **Everything above is verified by creation too.** For every combination `IsIndexKeyEncodable` accepts — + > every accepted order at version 0 plus the six orders with a version-1 table — a database LibRed + > synthesises in that order, indexed by ACE, holds keys matching LibRed's byte for byte. A wrong LCID + > cannot pass quietly: ACE indexes with whatever order that LCID really names. + + *Not yet handled:* **Irish 1084** and the **CJK** orders (above). **Six** version-1 collations are + implemented — General v1, Indic v1, Romanian v1, and Croatian / Bosnian / Serbian v1 sharing one table — + and those six are the only non-CJK orders that differ from General v1, so the rest are covered by falling + back to it. DAO writes version 0 for every LANGID it accepts, so a v1 database needs another authoring + route. - **GUID:** the start flag `0x7F`, then the 16 GUID bytes in **canonical string order** (i.e. `guid.ToString("N")` bytes — **not** the mixed-endian `.ToByteArray()` storage layout), split into two 8-byte halves by a constant `0x09` marker, and terminated by `0x08` — a fixed **19-byte** key. Data bytes equal to `0x08`/`0x09` need no escaping (every field is at a fixed offset). Verified byte-for-byte - against ACE (zeros, all-`FF`, sequential, and random GUIDs); ACE also opens a LibRed-written GUID-PK + against ACE; ACE also opens a LibRed-written GUID-PK table and seeks a row by its key. Encoded/decoded by `IndexKeyEncoder`/`IndexKeyDecoder`. Example: `01020304-0506-0708-090a-0b0c0d0e0f10` → `7F 0102030405060708 09 090A0B0C0D0E0F10 08`. **Descending** inverts every byte of the ascending key **except the `0x09` field marker** (kept constant @@ -940,14 +818,104 @@ this is the practical cost of General over General Legacy, invisible in the sche a **control byte**: `0x09` when a further chunk follows (a full 8-byte chunk with more data to come), otherwise the **real-byte count of this final chunk** (`0x01…0x08`; `0x08` for a full final chunk). The count `≤ 8 < 0x09`, so control values never collide. This is exactly the - GUID chunking generalised to any length: a 16-byte value is two chunks (`… 09 … 08`), and the old - fixed 4-byte MSysQueries.Order case is the single-chunk form `7F <4B> 00000000 04`. The trailing + GUID chunking generalised to any length: a 16-byte value is two chunks (`… 09 … 08`), and the fixed + 4-byte MSysQueries.Order key is the single-chunk form `7F <4B> 00000000 04`. The trailing length-terminator makes shorter values sort before longer ones that share a prefix (correct binary prefix order). **Descending** inverts every byte **except the `0x09` continuation markers** (mirrors - GUID): flag → `0x80`, data bytes and the terminator inverted, markers unchanged. Verified byte-for-byte - against ACE's `EverythingIsBytes` fixture (3/4/5/8/16-byte keys, single- and multi-chunk) by - re-encoding each stored key's row value; descending has no ACE fixture and is extrapolated from the - verified GUID descending (ordering-tested for internal consistency). `IndexKeyEncoder.EncodeBinaryChunked`. + GUID): flag → `0x80`, data bytes and the terminator inverted, markers unchanged. Ascending is verified + byte-for-byte against ACE-written keys (single- and multi-chunk); descending is **unverified** against ACE, + extrapolated from the verified GUID descending. `IndexKeyEncoder.EncodeBinaryChunked`. + +### 10.4a Entry removal — LibRed compacts a leaf harder than ACE does + +Removing entries leaves the two engines with **identical index content and byte-different pages**. LibRed +rewrites the leaf without the removed entries; ACE returns less of their space. + +> After the same `DROP TABLE`, the catalog index roots it touches hold the *same entries in the same order* +> under both engines, while their free-space fields diverge: +> +> | index root | before | ACE | LibRed | +> |---|---|---|---| +> | `MSysObjects.ParentIdName` | 3,046 | 3,064 | 3,136 | +> | `MSysObjects.Id` | 3,436 | 3,445 | 3,463 | +> | `MSysACEs.ObjectId` | 3,256 | 3,274 | 3,311 | +> +> Since the surviving entries then sit at different offsets, the pages differ widely despite agreeing on +> every entry. This is index maintenance, not a drop artefact — any `DELETE` removing entries does it. + +**Not currently treated as a defect.** The logical content matches, and ACE goes on writing into these very +indexes in a LibRed-compacted file — after a LibRed drop it inserts rows, allocates pages and adds catalog +entries normally. Packing tighter is also the conservative direction. What is *not* established is why ACE +leaves the space; if a scenario ever turns up that requires matching it byte for byte, this is the knob, and +matching would mean deliberately compacting **less**. + +### 10.4b The 510-byte index entry limit + +**ACE stores an index entry of at most 510 bytes as built.** At exactly 510 it comes back byte-for-byte; a +value that would need 511 comes back as 510: the first **508** bytes kept, and the rest replaced by a +two-byte **checksum over the bytes that were dropped**. That is why two long values sharing a 508-byte +prefix still sort apart instead of colliding. + +#### The checksum + +A 16-bit fold over the **discarded run** — every byte from offset 508 to the end of the untruncated key — +stored big-endian in the last two bytes. Each byte is XORed into the **high** half and the state is then +folded; the fold happens *between* bytes, so the last byte of the run contributes its XOR and nothing else: + +``` +crc = 0 +for each byte b of the discarded run: + crc ^= b << 8 + if b is not the last: crc = (crc >> 8) ^ T[crc & 0xFF] +``` + +with no initial value and no final XOR. The step table is + +``` +T[1<> 8) ^ T[(crc ^ b) & 0xFF]`, passing the byte **through** the table; no conventional CRC +framing reproduces it. + +The function is **affine over GF(2)** — tails differing in one byte give `L(0xA3) = CA03`, `L(0x13) = 6980`, +`L(0xB0) = A383`, and `CA03 ^ 6980 = A383` exactly — and **shift-invariant**, so a byte at distance *d* from +the end contributes `S^(d-1)` of itself whatever the message length. The eight table rows above follow from +those contributions. + +Equivalently, and how `JetIndexKeyChecksum` implements it: fold every byte but the last in the form +`crc = (crc >> 8) ^ T[crc & 0xFF] ^ b`, then XOR the last byte's `b << 8` into the result. The two are the +same function. + +**The discarded run is 3 bytes at minimum** (truncation triggers only above 510, and the run is +`length − 508`), and the rule is keyed to the run's *last byte*, not to a fixed offset in the key (verified +for runs of 3 through 13 bytes). + +Verified against ACE for Latin, accented and Han text under both sort orders; for composite keys ending in +`LONG`, `CURRENCY` and `DOUBLE`; and for keys whose dropped bytes contain an inline **word-sort record** — +ACE does not reposition that record when truncating, so it reconstructs exactly. **Nothing is refused**; +every key past the cap is truncated the way ACE truncates it. + +> The "fold between bytes" framing matters, and is easy to get wrong in a way no all-text test can catch. +> Writing the loop as "fold every byte except the last" — reading the skipped byte as the text terminator — +> gives the identical answer whenever that byte is `0x00`, which it always is when the key ends in text. The +> two readings part company the moment the last key column is numeric. See +> [`docs/design/index-key-checksum.md`](../design/index-key-checksum.md). + +The cap is on the **whole entry, not per column**: two 200-character text columns weigh about 404 bytes of +key each, comfortably under the cap individually, and ACE stores their combined entry hashed at 510. + +Because it limits **weights** rather than characters, the text it buys depends on collation and script — and +this is the practical cost of General over General Legacy, invisible in the schema: + +| | bytes per character | characters indexed in full | +|---|---|---| +| v0, Latin | 1 primary | **255** — the column limit is reached first | +| v0, accented / CJK | 2 | **254** | +| v1, Latin | 2 primary | **253** | +| v1, accented | 3 | **169** | +| v1, Han | 4 (`FD FF AW DW`) | **127** | ### 10.5 Insertion and splitting @@ -956,11 +924,10 @@ separator is the **maximum key of its child subtree**, stored as a full leaf key 4-byte row pointer), so descend into the first child whose separator `≥` the new full key, else the child-tail (`0x14`). Slot the new entry into the target leaf in key order and rewrite the page. -When a page would overflow, **split** it (LibRed's `IndexWriter`). Verified by inserting 1500 keys — -past one leaf — and reading every one back in order through a now multi-level tree, **and against ACE**: -Access opens the file and an indexed point seek (`WHERE Id = 1234`), an indexed range (`Id BETWEEN 300 -AND 309`), a full `COUNT(*)`, a non-indexed scan (`T LIKE 'r%'`) and `SUM(Id)` all return the correct -result — i.e. every row is reachable both by the tree and by the leaf-chain scan Access uses. +When a page would overflow, **split** it (LibRed's `IndexWriter`). Verified **against ACE** on a +multi-level tree LibRed wrote: Access's indexed point seek, indexed range, full `COUNT(*)`, non-indexed scan +and `SUM` all return the correct result — i.e. every row is reachable both by the tree and by the leaf-chain +scan Access uses. The split mechanics: @@ -978,28 +945,24 @@ The split mechanics: its child becomes the left node's child-tail, and the old tail stays the right node's tail. - **Right-edge split.** When the incoming key is the highest on the page, both engines leave that page full and start a new one holding the new entry alone, instead of halving it: nothing sorts below a maximum key, - so a middle split there strands half a page for ever. LibRed split down the middle unconditionally until - this was measured, and so spent about 1.8x the leaves on a sequential load — the ordinary case, since - AutoNumber and identity keys ascend by construction. Measured on 1500 rows through both engines (leaf free - space, sorted): + so a middle split there strands half a page for ever. A writer that always splits down the middle spends + about 1.8x the leaves on a sequential load — the ordinary case, since AutoNumber and identity keys ascend by + construction. Leaf count and free space (sorted) after 1500 rows: - | inserted | ACE | LibRed before | LibRed now | + | inserted | ACE | middle split only | with right-edge split (LibRed) | | --- | --- | --- | --- | | ascending | 3 leaves — `1, 1, 952` | 4 — `31, 1807, 1807, 1807` | 3 — `1, 1, 1837` | | descending | 4 — `31, 1807, 1807, 1807` | 4 — `49, 1801, 1801, 1801` | unchanged | | random | 4 — `1267, 1369, 1405, 1411` | 4 — `1291, 1357, 1387, 1417` | unchanged | The rule is **right-edge only**: descending inserts get an ordinary middle split from ACE too, and on - random keys both settle near two-thirds full — the classic B-tree equilibrium. Those two workloads are - what make the special case free: its condition cannot fire when the new key is not the page maximum, so - the general behaviour is untouched. They also showed LibRed's middle split already matched ACE's, which is - why this was an added case rather than a change to the split machinery. - - It also appears to cost nothing on the workload it is supposed to: the obvious objection — that a page - packed to capacity must split as soon as anything lands in its range — did not show up in a gapped load - backfilled ascending (ACE 5 leaves `1, 1, 1, 7, 55` against LibRed's then-6), because an ascending - backfill keeps meeting the right edge of a subtree. A *random* backfill into pre-packed pages has not been - measured. `IndexSplitPackingAccessTests`. + random keys both settle near two-thirds full — the classic B-tree equilibrium. Its condition cannot fire + when the new key is not the page maximum, so the general behaviour is untouched. + + The obvious objection — that a page packed to capacity must split as soon as anything lands in its range — + does not show up in a gapped load backfilled ascending (ACE: 5 leaves, `1, 1, 1, 7, 55`), because an + ascending backfill keeps meeting the right edge of a subtree. A *random* backfill into pre-packed pages is + unmeasured. - **Propagation:** the promoted separator `[key → left page]` is inserted into the parent, whose pointer to the just-split page is repointed to the new right page; if the parent overflows it splits in turn, up to the root. @@ -1014,16 +977,15 @@ The split mechanics: > [page-05 §9](page-05-usage-maps.md), which owns that rule. -> **Indexable types — coverage vs ACE (§10.4).** `IndexKeyEncoder` now encodes **every type ACE lets you +> **Indexable types — coverage vs ACE (§10.4).** `IndexKeyEncoder` encodes **every type ACE lets you > index**, all byte-verified: Boolean, Byte, Int16, Int32, Currency, Single, Double, DateTime, Text, GUID, > Binary, FixedPoint, Memo (its first 255 chars), **`Int64`/BIGINT** (`0x13`) and > **`DateTimeExtended`/DATETIME2** (`0x14`). ACE correctly **refuses** to index `OLE` (`0x0B`) and `Complex` > (`0x12`). > -> `Int64`/BIGINT keys exactly as Currency does — an int64, sign bit flipped, big-endian — which had long been -> the guess on record and is now measured across `0`, `±1`, `±42` and both extremes, ascending and descending -> (`BigIntKeyEncodingTests`). Note its **variable-length storage does not change this**: the key dispatch is on -> the column's type, not on where the row keeps the bytes. `IndexKeyDecoder` decodes it too, unlike DATETIME2 — +> `Int64`/BIGINT keys exactly as Currency does — an int64, sign bit flipped, big-endian — verified against ACE +> including both extremes, ascending and descending. Note its **variable-length storage does not change +> this**: the key dispatch is on the column's type, not on where the row keeps the bytes. `IndexKeyDecoder` decodes it too, unlike DATETIME2 — > it is a plain fixed-width numeric key. > > `DateTimeExtended` is **not** a fixed-width numeric key. ACE runs its whole 42-byte stored value through the @@ -1031,6 +993,6 @@ The split mechanics: > real-byte count — instead of folding it to a number the way `DateTime` folds to its OA double. That works > because the stored encoding is already order-preserving (both fields zero-padded to 19 digits), and it means > the value's trailing NUL is part of the key ([data-types](data-types.md)). Descending inverts every byte -> except the `0x09` markers, exactly as for Binary. Verified both directions in `DateTime2KeyEncodingTests`. +> except the `0x09` markers, exactly as for Binary. Verified against ACE in both directions. > `IndexKeyDecoder` does not decode it, for the same reason it does not decode Binary or Text: the chunked > form stops the in-place walk, and the caller falls back to reading the row. diff --git a/src/LibRed/docs/format/page-05-usage-maps.md b/src/LibRed/docs/format/page-05-usage-maps.md index aa37fc078..5c5e24b6f 100644 --- a/src/LibRed/docs/format/page-05-usage-maps.md +++ b/src/LibRed/docs/format/page-05-usage-maps.md @@ -36,7 +36,7 @@ each nonzero pointer must be within the physical file, and its target must carry `[05 01 00 00]` bitmap-page header. This validation happens before any bitmap is expanded into page numbers, so appended pointer-shaped bytes or pointers to ordinary data pages cannot become ownership data. -> Verified against an ACE-built 134 MB table (34,000 full-page rows): owned map record +> Verified against an ACE-built table large enough to need one: owned map record > `01 017D0000 E17F0000 00…` — type `0x01`, slot 0 → page 32,001, slot 1 → page 32,737, remaining > 15 slots zero, record length 69. Bitmap page 32,737's first bitmap byte is `0xFC`: pages 32,736 and > 32,737 clear (32,737 *is* the bitmap page), 32,738–32,743 set. @@ -47,14 +47,13 @@ numbers, so appended pointer-shaped bytes or pointers to ordinary data pages can > **LibRed write behaviour (multi-page growth on insert).** When an insert finds no owned data page > with room, LibRed allocates a new page (via the global map, §9.1), initialises it as an empty data > page owned by the table, sets its **owned** bit, and moves the **free** marker to it. Verified: -> 200 rows spill across ~20 data pages and Access reads the whole table and can still insert. +> Access reads a table LibRed spilled across many data pages and can still insert into it. > - **Free-pages map = the current append tail only.** Access clears a page from the free-pages map > when an insert finds it too full and moves on, so after a sequential fill only the *last* page -> stays marked free — verified: six equally-full ACE pages (all 252 bytes free) had only the last in -> the free map, because the earlier five each had a next-row attempt that didn't fit. LibRed matches -> this: on allocating a new page it **clears the previous tail's free bit and sets the new page's**, -> leaving exactly the tail marked free. (Non-sequential fills and deletes aren't specially handled — -> neither is supported yet.) +> stays marked free — verified: of equally-full ACE pages only the last is in the free map, because +> each earlier one had a next-row attempt that didn't fit. LibRed matches this: on allocating a new page +> it **clears the previous tail's free bit and sets the new page's**, leaving exactly the tail marked +> free. (Non-sequential fills and deletes aren't specially handled — neither is supported yet.) > - **Inline map, grown in place.** LibRed writes the inline map with `startPage = 0` and an initially > 64-byte bitmap (pages 0–511). When an insert needs to mark a page **past** that window, LibRed grows > the bitmap **record in place** — still type `0x00`, same `startPage` — extending it in **32-bit (4-byte) @@ -62,12 +61,11 @@ numbers, so appended pointer-shaped bytes or pointers to ordinary data pages can > from the end backward. So the record length is exactly > `5 + roundUp(ceil((maxPage + 1 − startPage) / 8), 4)`. > Verified against ACE owned-map lengths on a 255-column table whose data pages start at 353: -> 8,000 rows → **1053**, 12,000 → **1553**, 16,000 → **2053**, 30,000 → **3801**, 31,000 → **3925** — the -> formula reproduces every one. (An earlier reading of this as *256-bit / 32-byte* chunks fitted the one -> small data point it was drawn from — a table spanning to page 753 carries a **96**-byte bitmap, record -> length **101** — because 96 is a multiple of both. The large-table lengths discriminate: 32-byte rounding -> would give 1056 / 1568 / 3808.) Access opens a LibRed-grown table, counts every row, and reads one living -> past page 512. +> 8,000 data pages → **1053**, 12,000 → **1553**, 16,000 → **2053**, 30,000 → **3801**, 31,000 → **3925** — the +> formula reproduces every one. The step is **4 bytes, not 32**: a table spanning to page 753 carries a +> **96**-byte bitmap (record length **101**), which fits either because 96 is a multiple of both, but +> 32-byte rounding would give 1056 / 1568 / 3808 for the larger tables. Access opens a LibRed-grown table, +> counts every row, and reads one living past page 512. > - **Inline → reference conversion.** Access keeps growing the inline record until it no longer fits its > usage-map page, then rewrites the map as type `0x01`. Verified by owned-map record length against > page count: 8,000 pages → 1053, 12,000 → 1553, 16,000 → 2053, 30,000 → **3801, still type `0x00`**; @@ -75,8 +73,7 @@ numbers, so appended pointer-shaped bytes or pointers to ordinary data pages can > by a fixed page-count threshold. LibRed applies the same rule: grow inline while the repacked record > fits, otherwise convert — re-marking every previously-owned page into freshly allocated bitmap pages > (grouped by slot, one write per bitmap page) and shrinking the record to the fixed 69 bytes. -> ACE reads a LibRed-written reference map: a 255-column, 400,000-row, 126.8 MB table counts back -> exactly through `Microsoft.ACE.OLEDB.16.0`. +> ACE reads a LibRed-written reference map and counts every row back. > - **Movable window (free-pages maps).** A free-pages map's set bits stay clustered at the append tail, so > Access never grows it: it slides a fixed **64-byte bitmap (512 pages)** whose `startPage` is > `floor(page / 512) × 512`. Verified — a table whose tail page was 852 / 1227 / 1852 / 2852 had a free @@ -93,46 +90,133 @@ numbers, so appended pointer-shaped bytes or pointers to ordinary data pages can > | Table | Records on the map page | Owned-record budget | Converts at | > | --- | --- | --- | --- | > | No primary key | owned + free | `4096 − 14 − 4 − 69` = **4009** | page **32,032** | -> | Primary key | owned + free + the index's own map | `4096 − 14 − 6 − 69 − 69` = **3938** | page **31,456** | +> | Primary key | owned + free + the index's own map | `4096 − 14 − 6 − 69 − 69` = **3938** | page **31,456** while the index map stays 69 bytes | > -> Both verified end-to-end. LibRed's no-PK table converts at exactly page 32,032 with a last inline record -> of 4009 bytes; ACE's no-PK table is still inline at page 31,354 (record 3925) and reference by 32,356, -> bracketing the same value. With a primary key ACE likewise carries a third record and has already -> converted by page 31,409, against LibRed's 31,456. +> The no-PK row is verified end-to-end: LibRed's table converts at exactly page 32,032 with a last inline +> record of 4009 bytes; ACE's is still inline at page 31,354 (record 3925) and reference by 32,356, +> bracketing the same value. **The primary-key row is not reconciled.** ACE carries the third record but has +> already converted by page 31,409, earlier than 31,456 — consistent with the index's map growing past 69 +> bytes (below) and shrinking the budget, but the exact ACE conversion point with a key is not measured. > -> > Beware comparing thresholds across table shapes: an earlier note here claimed LibRed converted "at -> > ~31,000 pages vs Access at 34,000". Those were a **primary-keyed** LibRed table and a **key-less** ACE -> > one. The rule is identical; only the budget differed. A keyed table's index usage map also *grows* with -> > the index's pages (its own B-tree), further shrinking the owned map's budget — LibRed now matches this -> > (`IndexWriter` marks each index page it allocates), where it previously left the index map at 69 bytes. +> > Beware comparing thresholds across table shapes: a **primary-keyed** table and a **key-less** one +> > convert at different page counts under the identical rule, because only the budget differs. A keyed +> > table's index usage map also *grows* with the index's pages (its own B-tree), further shrinking the +> > owned map's budget — LibRed matches this (`IndexWriter` marks each index page it allocates). > **Owned-row recycle on an index rebuild (verified vs ACE, §3.8).** When ACE rebuilds an index (e.g. an > `ALTER COLUMN` on an indexed column) it gives the index a **new** owned-pages usage-map row rather than > editing the old one in place, in two steps whose leftover is observable on disk: **(1)** append a fresh row -> at the end of the usage-map data page and set the new root's bit; **(2)** **move** that map into the *old* -> row's freed slot (its row-directory entry now points there) and turn the **old** row into a **0-length -> deleted + overflow tombstone** — leaving the bytes at the *appended* slot **stale in free space** (never -> reclaimed). The index-data block's usage-map row field (`0x22`, §3.5) is re-pointed to the recycled row -> number. LibRed reproduces this exactly (`RecycleOwnedMapRow`), including the stale appended bytes, so the -> whole file matches ACE byte-for-byte. +> at the end of the usage-map data page and set the new root's bit — those bytes are then **abandoned** and +> stay behind, stale, in free space; **(2)** **re-lay** the live records with the *old* row's record +> **reclaimed**: its slot becomes a **0-length deleted + overflow tombstone** at the preceding record's +> offset, every later row **keeps its number** while its record slides up by the reclaimed width, and the +> fresh map takes the position freed at the end of the live region under the appended row number. The +> index-data block's usage-map row field (`0x22`, §3.5) is re-pointed to that number; **no other pointer +> changes**, because no other row's number does. LibRed reproduces this exactly (`RecycleOwnedMapRow`), +> abandoned bytes included, so the whole file matches ACE byte-for-byte. Measured on ACE's own page, records +> identified by content, with a long-value column's maps sitting below the index's: +> +> ``` +> before row2 @3889 pages=[353] row3 @3820 pages=[] row4 @3751 pages=[] free=3727 +> after row2 @3958 TOMBSTONE row3 @3889 pages=[] row4 @3820 pages=[] free=3725 +> row5 @3751 pages=[355] ← the new map, under the appended row number +> stale @3731 = 0x08 ← step (1)'s abandoned record, at 3682 +> ``` +> +> The long-value maps slid up a record width and kept rows 3 and 4; only the index's pointer moved, to row 5. +> +> > **Each half hides from a different measurement.** The abandoned copy lies *below* the lowest live +> > record, inside the region free space already covers, so slot offsets and free-space arithmetic both read +> > the page as though it were not there — yet dropping step (1) costs a byte against ACE. The re-lay is +> > invisible to offsets alone, because a moved record and a slid record occupy the same places; only +> > identifying records **by content** separates them. And the two are indistinguishable altogether when the +> > recycled row is the **last** one — the only kind an ACE-built schema produces, since a long-value column +> > declared in `CREATE TABLE` takes its map rows before the index's. So writing step (1)'s record into the +> > old row's slot passes every last-row shape, and then points a slot back **up** the page the moment a +> > Memo/OLE column is added *after* an index — which no reader can walk, a row's extent running to where the +> > previous slot begins. Only a whole-file byte diff against ACE catches both halves. + + +### 9.1 Global usage maps — free and released pages (page allocation) + +Besides the per-table maps, the database has two **global** usage maps, found through page 0 rather than the +catalog ([page-00 §2](page-00-database.md)): + +| Page 0 | Map | In every file ACE writes | +| --- | --- | --- | +| `0x18` | **free pages** — a set bit is a page available for allocation | page 1, row 0 | +| `0x1C` | **released pages** — a set bit is a page freed but not yet reusable | page 1, row 1 | + +Both are ordinary usage-map records (inline or reference form, §9) on a data page whose owner field +(`0x04`) reads `0x00000001`, each starting as a 69-byte inline map with start page `0`. In the free map a +**set bit means the page is free / available**, the *opposite* of a per-table owned map — verified by +diffing before/after an ACE `CREATE TABLE`. + +**ACE follows the pointers; the location is not fixed.** Allocation uses whichever record `0x18` names, row +included: pointed at page 1 row 1, ACE allocates from that map and leaves row 0 untouched. With both maps +copied to another data page and the pointers aimed there, ACE allocates, releases and reopens entirely on +that page and never reads or writes page 1; the holder page's owner field (`0x04`) is not checked. The two +pointers must name **different** records — naming the same one clears the free map when the released map +is emptied, and freed pages are lost. + +**Both pointers are range-checked on every open, row byte excepted.** A page past the end of the file opens +and reads once, then records `01 00` ("accessed a corrupted page", [page-00 §2.2](page-00-database.md)) in +the opening user's commit slot at close, and every later open fails with *"Unrecognized database format"*. +A far larger page number is refused on the first open — from page 524,289 exactly, as measured on `0x1C`. +Only the page is checked on open — any row byte passes — but a `0x18` naming a record that is not a usage +map (page 0, a TDEF page, an ordinary data page) fails at the first allocation and damages the database. + +**Freed pages are released at close, through the released-pages map.** Pages freed during a session are not +reusable on the same connection: later allocations in that session grow the file instead, and the free map on +disk does not change until the file closes. At close ACE moves them into the free map. This holds for the +long-value pages of a deleted row, the pages of a dropped index — by `DROP INDEX`, by `DROP CONSTRAINT` on a +foreign key, or by the index rebuild of an `ALTER COLUMN` — and every page of a dropped table. A free made in a +transaction that rolls back frees nothing; one that commits stays released even if a later transaction on the +connection rolls back. + +The exception is the long value an `UPDATE` replaces: its pages are set in the free map at once, and later +statements on the same connection reuse them; the `UPDATE` that frees them does not. + +Any page set in the released-pages map is likewise **never allocated**, and at close its pages are merged into +the free map and the map is left empty. Before that merge, the close sizes the map to cover every page released. +An inline record that already covers them stays as it is. Otherwise, in order of preference: +- **Lengthen it.** Keeping its start page, the record grows, never shrinks, to the shortest that covers the + highest page released: 5 header bytes plus `roundUp(⌈(highest + 1 − start) / 8⌉, 4)` bitmap bytes — page 569 + gives a 77-byte record, page 728 a 97-byte one. The record grows only while its holder keeps **4 bytes free**, + the same limit as the free map's growth below. +- **Move its window.** When that is too long, the start page becomes the lowest page released rounded down to a + multiple of 8, and the record is sized the same way from there — pages 32,819–33,825 released into a 69-byte + record starting at page 0 give start page 32,816 and a 133-byte record. +- **Convert it to reference form.** When even the moved window is too long: + 1. the inline record grows at its old start just far enough to cover the highest released page it can reach + — at most 4,005 bytes beside a 69-byte free map, covering 32,000 pages — and the released pages it covers + are set in it. Released pages running on past page 32,000 take it to the full 4,005 bytes; with released + pages only in the first and third ranges it stopped at 3,609; + 2. a bitmap page is allocated for each 32,736-page range the released pages fall in, in range order, from + the free map as it stands before the released pages are merged into it — the first page past the end of + the file when nothing is free. A range holding no released page gets none, even between two that do; + 3. a 69-byte reference record naming them replaces the inline one. The records are repacked from the page + end and the vacated bytes are not cleared, so the long record's bitmap stays on the page below the new + one. -### 9.1 Global free-pages map — page 1 (page allocation) +A map already in reference form gains a bitmap page, allocated the same way, for each range holding a released +page that it has none for — a released table-definition page among the free pages is taken like any other, its +type byte becoming `0x05`. The merge then clears every bitmap page, each keeping its `05 01 00 00` header. -Besides the per-table maps, the database has a **global free-pages map** at **page 1, row 0** (a -data page; its row 0 starts as an inline usage map, start page `0`). Here a **set bit means the page is -free / available**, the *opposite* of a per-table owned map — verified against Northwind (161 free -pages among 353) and by diffing before/after an ACE `CREATE TABLE`. +So at rest the released-pages map has no bits set, though it may have grown, moved its start page or converted +to reference form. A non-empty one is *inferred* to be a release interrupted before close. -**Page allocation works through this map.** Access does **not** simply grow the file: it finds a +**Page allocation works through the free-pages map.** Access does **not** simply grow the file: it finds a set bit (a free page), **clears it** (marking the page used), and reuses that page — only growing -the file when no free page remains. Verified: creating a table in Northwind reused four free pages -(for the TDEF, usage map, etc.) and grew the file by a single page; the only change to page 1 was -one cleared bit per page taken. +the file when no free page remains. Verified: an ACE `CREATE TABLE` reuses free pages (for the TDEF, +usage map, etc.), and the only change to page 1 is one cleared bit per page taken. -> LibRed allocates **through** this map (`PageAllocator`): it takes a free page, clears its bit, -> and reuses it — only growing the file when none is free — so its pages now match Access's -> allocation. Free bits at the current file end are the pre-allocated growth region; LibRed materializes +> LibRed allocates **through** this map (`PageAllocator`), found as ACE finds it — through page 0's `0x18` +> pointer, row included — and never takes a page set in the released-pages map named at `0x1C`. A released +> page at the end of the file is materialized, so the file stays contiguous, but not handed out. It takes a +> free page, clears its bit, and reuses it — only growing the file when none is free — so its pages match +> Access's allocation. Free bits at the current file end are the pre-allocated growth region; LibRed materializes > that next page contiguously before returning it, and rejects a bit that would skip beyond it. Repeated > allocations can therefore consume an ACE-authored run of future bits without creating a sparse file. > **Both map forms are handled.** For an inline (`0x00`) map it scans the record's @@ -142,25 +226,79 @@ one cleared bit per page taken. > `Free` is the inverse (sets the bit). A page outside a pre-existing map's coverage cannot be recorded > as free until that coverage exists. > +> **Release at close.** The frees ACE holds go through `Release`, which keeps the page in a list on the handle +> — staged with the open transaction, kept on commit, dropped on rollback or on a rollback to a savepoint +> taken before it. Closing a writable `JetDatabase` returns those pages and any already set in the +> released-pages map to the free map, clears the released map and first sizes it as above, all in one +> transaction. A close that released nothing and wrote nothing writes +> nothing. Only an `UPDATE`'s replaced long value goes through `Free` at once. +> +> **Where LibRed differs from ACE.** An `UPDATE` frees the old long value before writing the new one, so the +> new value reuses those pages in the same statement. Held pages live in the handle, not in the released-pages +> map, so every handle releases its own at its own close, even while other handles are open. +> > **Global-map growth.** The inline growth rule in §9 applies, but ACE leaves **4 bytes free in the > holder page** before promoting the global map to reference form. With a 69-byte companion row, > the final inline record is 4005 bytes, covering 32,000 pages. LibRed matches this transition and > allocates each required bitmap page before the data page, marking the bitmap itself used. > Existing file pages are marked used and the remaining new coverage free. > -> **Allocator mutation guardrails.** Page 1 must be a valid data page with a live, non-overflow row 0. -> Inline records require their complete header; reference records require exactly 69 bytes, unique in-file -> bitmap pointers, and the complete `[05 01 00 00]` header. Pages 0/1, bitmap pages themselves, out-of-file -> free targets, and non-contiguous growth targets are rejected before a free bit is cleared or set. -> -> **Create-table side effects.** An ACE `CREATE TABLE` *also* (1) adds two rows to **`MSysACEs`** -> (the new object's permission entries) and updates its `ObjectId` index, and (2) bumps the opening user's -> commit counter at `0xE02`. **(1) is now done** — -> `TableCreator.AddPermissionRows` writes both permission rows (§11), and ACE opens LibRed-created -> tables without repair (`CreateTableAccessTests`). **(2) appears not to be required:** LibRed does not -> touch the page-0 counter, yet ACE opens/queries the created tables — so it's either unused for -> table open or benign when stale. (Views likewise get their two `MSysACEs` rows now — §11.) +> **Allocator mutation guardrails.** Both page-0 pointers must name distinct, live, non-overflow rows on +> data pages inside the file, each an inline record with its complete header or a reference record of +> exactly 69 bytes with unique in-file bitmap pointers and the complete `[05 01 00 00]` header. A writable +> open checks this before anything else; a read-only open, which never allocates, does not. Page 0, the +> map holder pages, bitmap pages themselves, out-of-file free targets, and non-contiguous growth targets are +> rejected before a free bit is cleared or set. + +**Releasing a table means walking every map it owns, not just the data-page one.** A Memo/OLE (or +calculated long-value) column holds its LVAL pages in a **per-column owned map**, whose (row, page) pointer +sits in the TDEF keyed by column id — those pages never appear in the table's own data-page owned map. A +`DROP TABLE` that frees only the data pages therefore strands the entire content of the table. Likewise each +index holds every page of its B-tree — root, intermediate and leaf — in its own owned map, whose pointer sits +in the index's data block (`0x22`); ACE frees them all. + +> Measured. A table of 3,000-character memo values can occupy many pages while its data-page map names +> **one**: the row records, each holding a 12-byte descriptor, fit on a single page while the text lives on +> LVAL pages. Dropping it through ACE returns all of them to the global free map at close, and refilling the +> file reuses the space; freeing only the data pages and the TDEF returns almost nothing, and a refill grows the +> file instead — the engine reuses exactly what the global map offers it, and nothing else. + +**A map's records are retired from their holder, and the holder goes back once nothing else lives on it.** +The records are rows on owner-zero data pages, and one holder can carry records for several columns or +tables. Dropping a table retires each of its map records in turn: it clears the freed pages' bits where it +clears them, tombstones the record's row, and — once no live row is left — frees the holder page itself. +Dropping a memo/OLE column retires that column's two records the same way — step 1 below, for that column +alone ([long-values](long-values.md#dropping-a-long-value-column)). + +Each tombstone slides the records below it up the page, and the bytes they vacate are not cleared, so a moved +record leaves a copy of itself behind. The order is therefore visible on disk, and ACE's is fixed: + +1. each long-value column's owned map, then its free map — bits cleared, except for a page still in the + column's free map (its current append page), whose bit stays set in both records; +2. each index's owned map, in index order — bits cleared; +3. the table's own data owned map — bits cleared — then its free map — bits left set. + +A map in reference form has its record retired in the same order; each of its bitmap pages has its bitmap +zeroed and is freed, its `05 01 00 00` header left in place. + +> Measured against an ACE drop, page by page. ACE clears the bitmap bytes of the record as it frees each +> page (the zeroing is visible inside the space the row then gives up), tombstones each 69-byte map record +> with a `0xD000` slot, raises the page's free-space field by the bytes they occupied, and returns the page. +> Doing all three, in the order above, makes the holder **byte-identical** between the two engines — +> including for an indexed table whose second index's record sits below the long-value maps and slides +> twice; returning the page alone leaves it differing. > -> That counter is **no longer undecoded**: it is a 16-bit little-endian count of the user's committed writes, -> and it is not specific to `CREATE TABLE` — every committed write moves it, reads never do. See -> [page-00 §2.2](page-00-database.md#the-slot-is-a-little-endian-commit-counter-verified-2026-08-26). +> The exclusivity test matters — releasing a holder that still carries another map's row would hand away a +> live page, which is corruption rather than a leak. It is also the case that *only* clearing the bits is +> not enough on a shared holder: the row has to go, or the dropped table's map records outlive it. + +**The released definition page is marked.** Access sets the dropped table's TDEF page type to **`0x08`** +and changes nothing else on it; the other pages a drop frees (data, long-value, map holders, bitmap pages, and +a wide definition's continuation pages) keep their original type bytes. The marker, what survives on the page +and how close a LibRed drop lands to an ACE one are in [page-08](page-08-released-tdef.md). + +#### A long-value page is released on its own terms + +Deleting the last value that shared a packed long-value page releases the page and stamps it **`0x09`**, +clearing it from the column's owned and free maps here. That mechanism, and the page it leaves behind, are in +[page-09](page-09-released-long-value.md). diff --git a/src/LibRed/docs/format/page-08-released-tdef.md b/src/LibRed/docs/format/page-08-released-tdef.md new file mode 100644 index 000000000..d1cf76a97 --- /dev/null +++ b/src/LibRed/docs/format/page-08-released-tdef.md @@ -0,0 +1,42 @@ +# Page type `0x08` — a released table-definition page + +A TDEF page that `DROP TABLE` has given back. Access marks it by **setting this one byte and changing +nothing else**: the dropped table's definition — header, column descriptors, names, index blocks — stays on +the page exactly as it was until a Compact reclaims it. + +> Measured across an ACE `DROP TABLE`: **exactly one byte of the TDEF page's 4,096 changes**, offset `0x000`, +> `0x02` → `0x08`. + +That is what the `0x08` pages in real-world files are, and it is why they read as *structured* rather than +blank — a reader walking raw pages finds a whole table definition sitting behind a type byte that says the +page is no longer in use. It is also the reason a released TDEF cannot be mistaken for a live one by type +alone. + +The other pages a drop frees — data pages, long-value pages, usage-map holders — **keep their original type +bytes**; only the definition page is marked. A definition that runs onto continuation pages has only its first +page marked: the continuations are freed with every byte, their `0x02` type included, left as it was. (A long-value page released for a different reason does get its +own marker, [`0x09`](page-09-released-long-value.md), but not as part of a drop.) + +## Reading + +**Nothing needs to handle it.** Allocation selects on the global free map, not on this byte, so a released +TDEF is handed out and overwritten like any other free page. LibRed names it +`PageType.ReleasedTableDefinition` so that a page walk can report it, and treats it no further. + +## Writing + +LibRed sets the same marker in `TableCreator.DropTable`, as the last step of giving the table's pages back +(the owned/free maps, the per-column long-value maps, and the map-holder rows are covered in +[page-05 §9](page-05-usage-maps.md)). + +> An ACE drop and a LibRed drop of the same table leave the file **byte-identical** except for the three +> catalog **index root pages** — identical entries in identical order, but LibRed compacts a leaf harder than +> ACE does after removing entries ([page-03-04 §10.4a](page-03-04-index-btree.md)) — and page 0, by the one +> byte of the opening user's commit slot at `0xE02` ([page-00 §2.2](page-00-database.md)), which moves for any +> write at all. + +## Related + +- [page-02a](page-02a-tdef.md) — what the page held while it was live. +- [page-05 §9](page-05-usage-maps.md) — the rest of what a drop gives back. +- [page-09](page-09-released-long-value.md) — the other released-page marker, from a different mechanism. diff --git a/src/LibRed/docs/format/page-09-released-long-value.md b/src/LibRed/docs/format/page-09-released-long-value.md new file mode 100644 index 000000000..45f721663 --- /dev/null +++ b/src/LibRed/docs/format/page-09-released-long-value.md @@ -0,0 +1,63 @@ +# Page type `0x09` — an emptied packed long-value page + +A long-value page whose last tenant was deleted. Values in the **single-page form** (≤ 3,816 bytes — see +[long-values](long-values.md)) are packed several to an LVAL page; each delete retires that value's row and +re-lays the page, and when no live record is left the page is stamped `0x09` and given back. + +They are **released, not orphaned**: the bit is set in the global free map, so ACE reuses them and Compact +reclaims them. Structurally the page is an emptied data page — every row slot a **0-length deleted + overflow +tombstone**, the rest free — still carrying the `LVAL` signature at offset `0x04`, because that is what it was. + +| Offset | Size | Value on a released page | +| --- | --- | --- | +| `0x00` | 1 | `0x09` | +| `0x01` | 1 | `0x01` (page flags, unchanged) | +| `0x02` | 2 | Free space — `PageSize − 14 − 2N` for `N` slots, i.e. everything below the directory | +| `0x04` | 4 | `LVAL` (`0x4C41564C`), as when it was live | +| `0x08` | 4 | Zero — the chain stamp is only set on a chain's first page ([long-values](long-values.md)) | +| `0x0C` | 2 | Row count `N` — how many values had shared the page; they all remain, as tombstones | +| `0x0E` | 2×N | Slot directory, every entry `0xD000`: deleted + overflow, offset `0x1000` (0-length) | + +## What produces one + +Deleting the **last** value sharing the page. Each delete tombstones that value's row and re-lays the page, +the survivors packing from the page end in slot order; the page type changes only when nothing live is left. + +> Measured against ACE — one LVAL page of five packed 400-character memos, as its values are deleted: +> +> ``` +> start 0x01 n=5 free=72 [3296,2496,1696,896,96] +> 4 gone 0x01 n=5 free=3272 [4096DO,4096DO,4096DO,4096DO,3296] the survivor slid to the top +> all 0x09 n=5 free=4072 [4096DO x5] +> ``` +> +> It happens through OLE DB SQL, DAO SQL and a DAO recordset alike, so no Access UI is involved, and exactly +> one page is released per page emptied. + +**A chained value never produces one.** Those own their pages outright and are freed at `0x01`. + +> These pages appear at **every format version** — Jet 4, ACE 12, ACE 14, ACE 16 — including in Jet-4-era +> files, so the mechanism long predates ACE. Their presence tracks a file's *history* rather than its format: +> heavily-edited applications hold hundreds, while freshly created or untouched files hold none. Some `0x09` +> pages in real files do **not** carry the `LVAL` signature; those are not yet accounted for. + +## Reading + +**Nothing needs to handle it.** Reading is unaffected — no live descriptor points at a released page — and +allocation selects on the free map without consulting the type byte, so one can be handed out and overwritten +normally. LibRed names it `PageType.ReleasedLongValuePage` so a page walk can report it. + +## Writing + +LibRed does the same, in `RowInserter.ReleasePackedValue`: tombstone the value's row, re-lay the page, +and when the last one goes set the type, clear the page from the column's owned and free maps, and return it +to the allocator. A page that survives goes back into the column's **free** map, having room again. + +Verified byte-for-byte, every page, against ACE's own delete for the partial, full, multi-page and chained +cases. + +## Related + +- [long-values](long-values.md) — the three storage forms, and which one packs. +- [page-05 §9](page-05-usage-maps.md) — the per-column owned/free maps this page is cleared from. +- [page-08](page-08-released-tdef.md) — the other released-page marker, from a different mechanism. diff --git a/src/LibRed/docs/format/system-catalog.md b/src/LibRed/docs/format/system-catalog.md index 5ae41eebe..46c7f430a 100644 --- a/src/LibRed/docs/format/system-catalog.md +++ b/src/LibRed/docs/format/system-catalog.md @@ -18,7 +18,7 @@ > creating the model's tables. Real user tables carry `Flags = 0x00000000`, so excluding the > system/hidden bits never drops a genuine table. - **Writing a table object** (verified against Northwind rows). A complete user-table row sets: + **Writing a table object** (verified against Access-written rows). A complete user-table row sets: `Id` = TDEF page; `ParentId` = `0x0F000001` (the database's "Tables" container, constant); `Type` = `1`; `Name`; `Flags` = `0`; `Owner` = a 2-byte binary SID (`0x69 0x0C` for a workgroup-less database, constant across tables); and `DateCreate` / `DateUpdate`. The other @@ -26,9 +26,9 @@ an OLE long-value blob ("MR2"-prefixed) holding the object's **extended properties** — including column-level properties such as *Required* (see §3.4) and *DefaultValue*. - > **Permission rows (`MSysACEs`) — one per object, verified against Northwind.** Every new object needs + > **Permission rows (`MSysACEs`) — one per object, verified.** Every new object needs > `MSysACEs` rows or Access warns about permissions when opening it (a **table** still opens; a **query** - > opens but pops a permissions warning). The table has exactly **four columns** (verified vs Northwind): + > opens but pops a permissions warning). The table has exactly **four columns**: > `ObjectId` (Int32, the object's id), `SID` (Binary, a security id), `ACM` (Int32, an access mask), and > `FInheritable` (Boolean). Each row sets `ObjectId` = the object id, `SID` = a 2-byte binary security id, > `ACM` = an access mask, `FInheritable` = false, and the object's `ObjectId` index must be maintained so @@ -50,14 +50,13 @@ > or `""` for the table): `[short ownerRecLen][short 0][short nameLen][owner name]` then property > entries `[short entryLen][byte DDL flag][byte dataType][short nameIndex][short valueLen][value]`. > The per-entry flag is `0x01` for a **DDL/property-definition property** and `0x00` for an ordinary - > property. Jackcess independently names it `isDdl`: a set flag makes the property definition-protected - > (`dbSecWriteDef` permission is needed to change/delete it), and it notes that Access only recognises - > some properties when the classification is correct. The fixture corpus matches that semantic: - > `DefaultValue`, `Required`, `CheckConstraints`, `GUID`, and `ResultType` are `0x01`, while `Title`, - > `Author`, `AccessVersion`, and datasheet-layout properties are `0x00`. Jackcess's built-in classifications - > additionally mark `ValidationRule`/`ValidationText` as DDL and `Caption`/`Description` as ordinary. - > The flag is independent per entry; it is not a file-version, encryption, owner, or data-type marker. - > LibRed accepts the two observed values, preserves both the flag and raw value read for every property, + > property. A set flag makes the property definition-protected (`dbSecWriteDef` permission is needed to + > change/delete it), and Access only recognises some properties when the classification is correct + > (both unverified against ACE). Observed in files: `DefaultValue`, `Required`, `CheckConstraints`, `GUID`, + > and `ResultType` are `0x01`, while `Title`, `Author`, `AccessVersion`, and datasheet-layout properties + > are `0x00`. `ValidationRule`/`ValidationText` are classed as DDL and `Caption`/`Description` as ordinary + > (unverified). The flag is independent per entry; it is not a file-version, encryption, owner, or + > data-type marker. LibRed accepts the two observed values, preserves both the flag and raw value read for every property, > and defaults newly constructed schema properties to `0x01`. The > `dataType` is an ordinary **`JetDataType` code** (the same byte used by column descriptors and > MSysQueries): **`0x0C`** (Memo) for a text value stored as **UTF-16**, **`0x01`** (Boolean) for a single @@ -65,12 +64,11 @@ > (Byte/Int16/Int32). The value-block **type** is `0x01` for a column-owned map and `0x00` for the > table-owned map (empty owner name). A `DefaultValue` (column property) is the expression's **source > text** (e.g. `42`, `'hi'`) — its evaluation semantics (what an expression may contain, the - > DDL-parser-vs-expression-service split) are in [page-02c-default-values.md](page-02c-default-values.md); table-level - > `CHECK` constraints are a single **table** property named - > `CheckConstraints` whose value is a `name\0expression\0` list, terminated by an extra `\0` (verified - > byte-for-byte vs ACE for `CONSTRAINT CK_BD CHECK ([BirthDate] < NOW())`). `ALTER TABLE … DROP CONSTRAINT - > ` removes the matching entry from that list and rewrites it (dropping the whole table-level property - > block when it was the last check) — ACE-verified: after the drop ACE stops enforcing the check. (In + > DDL-parser-vs-expression-service split) are in [page-02c-default-values.md](page-02c-default-values.md); + > table-level `CHECK` constraints are a single **table** property named `CheckConstraints` whose value is a + > `name\0expression\0` list, terminated by an extra `\0` (verified byte-for-byte vs ACE). + > `ALTER TABLE … DROP CONSTRAINT ` removes the matching entry from that list and rewrites it (dropping + > the whole table-level property block when it was the last check) — ACE-verified: after the drop ACE stops enforcing the check. (In > Jet/ACE `DROP CONSTRAINT` is polymorphic over the name — FK / PK / unique index / CHECK.) > > The `MSysDb` object — an `MSysObjects` row of `Type=2` with no table behind it — carries the @@ -88,14 +86,13 @@ > bytes; unmodelled `RawValue` payloads still round-trip verbatim. > > **`Required` (NOT NULL)** is a per-column **boolean** property (`dataType 0x01`, one `0x01` byte); a - > **nullable** column simply has **no** `Required` property, and an AutoNumber column is left without one - > too (verified vs ACE). Within a column's map ACE orders `DefaultValue` **before** `Required`; the - > name-pool order follows first appearance across all properties — **not** alphabetical. Verified with a - > deliberately non-alphabetical discriminator: a table whose names first appear as `Required` (a NOT NULL - > column), then `DefaultValue` (a later `DEFAULT` column), then `CheckConstraints` (an added CHECK) stores - > the pool in exactly that `["Required","DefaultValue","CheckConstraints"]` order — alphabetical would be - > `["CheckConstraints","DefaultValue","Required"]`. Example (`Req int NOT NULL, …, Def int DEFAULT 7 NOT - > NULL`): name pool `["Required","DefaultValue"]`, then `Req`'s `Required`, then `Def`'s `DefaultValue`=`7` + > **nullable** column simply has **no** `Required` property. An AutoNumber column follows the same rule: one + > declared `NOT NULL` (`COUNTER NOT NULL`, `INT NOT NULL IDENTITY`) carries `Required`, and one declared + > without it has none (verified vs ACE). Within a column's map ACE orders `DefaultValue` **before** `Required`; the + > name-pool order follows first appearance across all properties — **not** alphabetical (verified): names + > first appearing as `Required`, then `DefaultValue`, then `CheckConstraints` are pooled as + > `["Required","DefaultValue","CheckConstraints"]`, not `["CheckConstraints","DefaultValue","Required"]`. + > Example (`Req int NOT NULL, …, Def int DEFAULT 7 NOT NULL`): name pool `["Required","DefaultValue"]`, then `Req`'s `Required`, then `Def`'s `DefaultValue`=`7` > and `Required` — reproduced byte-for-byte by `PropertyBlob.Write` (which builds the pool by first > appearance via `Distinct()`). > @@ -103,22 +100,22 @@ > `CheckConstraints` and the text `ValidationRule`/`ValidationText` (the last two are **read-only**: surfaced > through `INFORMATION_SCHEMA.{TABLES,COLUMNS}.VALIDATION_RULE/VALIDATION_TEXT` to match EFCore.Jet's ADOX > `Jet OLEDB:{Table,Column} Validation Rule/Text`, but not yet written or enforced), while a database-first - > file may carry many more per column (`Format`, `AllowZeroLength`, the numeric `DecimalPlaces`, …). An ALTER that edits one property rewrites the whole - > blob (`PropertyBlob.Read` → mutate → `Write`), so `PropertyBlob.Property` keeps each value's **exact stored - > bytes** (`RawValue`) and re-emits them unchanged — a property LibRed doesn't model is never dropped or - > corrupted by the best-effort UTF-16 value decode (which would mangle a numeric one). `PropertyBlobRoundTripTests`. + > file may carry many more per column (`Format`, `AllowZeroLength`, the numeric `DecimalPlaces`, …). An + > ALTER that edits one property rewrites the whole blob (`PropertyBlob.Read` → mutate → `Write`), so + > `PropertyBlob.Property` keeps each value's **exact stored bytes** (`RawValue`) and re-emits them unchanged — + > a property LibRed doesn't model is never dropped or corrupted by the best-effort UTF-16 value decode (which + > would mangle a numeric one). > > LibRed **writes** `DefaultValue`, `Required` and `CheckConstraints` properties (`PropertyBlob.Write`) and > **reads** them back (`ColumnDef.DefaultValue`, `ColumnDef.IsNullable`, `TableDef.CheckConstraints`), > applying the default when an insert omits the column and **rejecting** an insert that leaves a required > column null ("You must enter a value in the '.' field.", matching Access). Access > **applies the default**, **enforces Required**, and **enforces the CHECK** on its own inserts — - > including on a LibRed-created table (verified: ACE rejects an insert omitting a LibRed `NOT NULL` column). `LvProp` is stored - > on a **single LVAL page** (`LongValueWriter`, descriptor flag `0x40`) — the form Access's property - > loader requires. **Verified:** Access opens the file and **applies the default** on its own insert - > that omits the column. (An *inline* value, flag `0x80`, is written and read fine by LibRed but is - > **not** recognised by Access's property loader — established by dumping the raw descriptors; nothing - > else differs, only `MSysObjects`+`MSysACEs` are touched.) + > including on a LibRed-created table (verified: ACE rejects an insert omitting a LibRed `NOT NULL` + > column). `LvProp` is stored on a **single LVAL page** (`LongValueWriter`, descriptor flag `0x40`) — the + > form Access's property loader requires. **Verified:** Access opens the file and **applies the default** on + > its own insert that omits the column. (An *inline* value, flag `0x80`, is valid long-value storage but is + > **not** recognised by Access's property loader.) > > **`ALTER COLUMN … SET DEFAULT expr` / `DROP DEFAULT`** are LvProp edits only — no TDEF/type change. Both > read the `LvProp` blob, mutate the target column's map, and rewrite it: SET replaces (or adds) that @@ -147,14 +144,14 @@ > LibRed only sets the descriptor's `0x04` AutoNumber flag and the header seed/increment (`0x14`/`0x18`); the > existing values are untouched, no rebuild. ACE rejects the conversion outright (*"Invalid field data > type"*, as does SQL Server); PostgreSQL (`ADD GENERATED AS IDENTITY`) / MySQL (`MODIFY … AUTO_INCREMENT`) / - > LibRed allow it. Round-trip verified: ACE reads the promoted counter and assigns next id = seed. Guards: + > LibRed allow it. Verified: ACE reads the promoted counter and assigns next id = seed. Guards: > Jet permits only one AutoNumber per table (a second is rejected), and a column in a relationship is > rejected (matching ACE). > > **Demoting a counter to a plain int** (`ALTER COLUMN LONG`) is the reverse in-place edit — clear > the `0x04` flag and reset the header to a non-AutoNumber table's state (`0x14` = 0, `0x18` = 1); values are > kept and the column stops auto-assigning. Unlike promotion this is **not** a divergence: ACE allows it too - > (matching the Access UI's AutoNumber→Number change). Round-trip verified: ACE reads the demoted column as a + > (matching the Access UI's AutoNumber→Number change). Verified: ACE reads the demoted column as a > plain int and accepts explicit ids. > > **Default-value interaction** (a "Random" AutoNumber *is* a counter with a `GenUniqueID()` default, so the @@ -170,24 +167,21 @@ > (descriptor flag `0x04`, TDEF `0x14`/`0x18` at their plain-counter defaults `0`/`1` and **ignored**) plus a > **column `DefaultValue` extended-property** holding the built-in expression **`GenUniqueID()`** — the > function that returns a random Long. There is **no** dedicated flag or "New Values" property; the - > Increment-vs-Random distinction lives entirely in this default expression. Verified against a modern - > Office-365-authored file (`Table1(ID AutoNumber, New Values=Random)`): the ID descriptor and TDEF header are - > **byte-identical** to an increment counter, and LibRed already surfaces it (`ColumnDef.DefaultValue` = - > `"GenUniqueID()"`) via the ordinary DefaultValue read path — no special handling needed to detect it. A - > Random AutoNumber **can** be created in pure SQL (not UI/DAO-only): `CREATE TABLE T (Id COUNTER DEFAULT - > GenUniqueID(), ...)` — also `AUTOINCREMENT`/`COUNTER PRIMARY KEY` forms — is accepted by ACE and yields - > genuinely random signed-Long IDs on insert (verified: `-1637443712, 1680187777, 83315118`), reading back - > byte-identical to the UI-authored column. `GenUniqueID()` **is a real ACE default-expression**, not a marker: - > `SELECT GenUniqueID()` - > errors ("Undefined function"), yet an **unquoted** `col LONG DEFAULT GenUniqueID()` **is** accepted and - > generates a **random signed Long per row** (verified: `117617513`, `904519542`, `-1470084161`). It is - > accepted **only on a `LONG` (Int32) column** — the same width a `COUNTER` stores; **every other type is - > rejected** ("Cannot place this validation expression on this field"), verified across BYTE/SHORT/SINGLE/ - > DOUBLE/CURRENCY/DECIMAL/GUID/DATETIME/BIT/TEXT. Quoting it — - > `DEFAULT 'GenUniqueID()'` — makes it a plain literal string stored verbatim. So a Random AutoNumber is - > effectively an AutoNumber column carrying the unquoted `GenUniqueID()` default. **LibRed now creates and - > inserts these**: `CREATE TABLE ( Id COUNTER DEFAULT GenUniqueID(), … )` persists the `GenUniqueID()` default - > to the column's LvProp (byte-identical to a UI/ACE-authored one, so ACE reads it as a Random AutoNumber), and + > Increment-vs-Random distinction lives entirely in this default expression. Verified against an + > Access-authored file: the ID descriptor and TDEF header are **byte-identical** to an increment counter, so + > the ordinary DefaultValue read path surfaces it (`ColumnDef.DefaultValue` = `"GenUniqueID()"`) with no + > special handling. A Random AutoNumber **can** be created in pure SQL (not UI/DAO-only): `CREATE TABLE T + > (Id COUNTER DEFAULT GenUniqueID(), ...)` — also `AUTOINCREMENT`/`COUNTER PRIMARY KEY` forms — is accepted + > by ACE and yields genuinely random signed-Long IDs on insert (verified), reading back byte-identical to the + > UI-authored column. `GenUniqueID()` **is a real ACE default-expression**, not a marker: `SELECT + > GenUniqueID()` errors ("Undefined function"), yet an **unquoted** `col LONG DEFAULT GenUniqueID()` **is** + > accepted and generates a **random signed Long per row** (verified). It is accepted **only on a `LONG` + > (Int32) column** — the same width a `COUNTER` stores; **every other type is rejected** ("Cannot place this + > validation expression on this field"), verified for BYTE/SHORT/SINGLE/DOUBLE/CURRENCY/DECIMAL/GUID/ + > DATETIME/BIT/TEXT. Quoting it — `DEFAULT 'GenUniqueID()'` — makes it a plain literal string stored + > verbatim. So a Random AutoNumber is effectively an AutoNumber column carrying the unquoted `GenUniqueID()` + > default. **LibRed creates and inserts these**: `CREATE TABLE ( Id COUNTER DEFAULT GenUniqueID(), … )` + > persists the `GenUniqueID()` default to the column's LvProp (byte-identical to a UI/ACE-authored one, so ACE reads it as a Random AutoNumber), and > on insert LibRed assigns a random non-zero Int32 per row instead of the sequential counter, leaving the TDEF > high-water (`0x14`) unadvanced (as ACE does). `ColumnDef.IsRandomAutoNumber` gates this off the default text. > A **plain (non-AutoNumber) `LONG DEFAULT GenUniqueID()`** column works too: `GenUniqueID()` is a real @@ -213,11 +207,26 @@ - **Views / queries** are `MSysObjects` rows of **Type 5** with a **negative synthetic `Id`** (queries increment from `0x80000000`), `ParentId 0x0F000001`, `Flags 0x10000000`, `LvProp` null. - > **MSysQueries columns (8, verified vs Northwind).** The table has exactly: `ObjectId` (Int32, the +- **Relationships** are `MSysObjects` rows of **Type 8** too — one per relationship, alongside its + `MSysRelationships` rows (verified vs ACE: every relationship Access or ACE creates has one, whether from + `ALTER TABLE … ADD CONSTRAINT` or a `CREATE TABLE` foreign key). `Name` is the relationship's name, + `ParentId 0x0F000003` (the Relationships container), `Flags 0`, `Owner 0x690C`, `DateCreate` = `DateUpdate` = + the creation time, and `LvProp`, `Lv`, `LvExtra`, `LvModule`, `Connect`, `Database`, `ForeignName`, + `RmtInfoShort`, `RmtInfoLong` all null. + - **`Id`** is the next negative synthetic id: one past the highest in the file, from the sequence queries draw + on, so relationships and queries interleave (`0x8000002C` relationship, `0x8000002D` view, + `0x8000002E` relationship), and a dropped relationship's id is taken by the next object. + - **Two `MSysACEs` rows**: SID `0x690C` with ACM `0xF00FE`, and SID `0x680C` with ACM `0xFFFFF`. + - **Dropping it** — `DROP CONSTRAINT`, or `DROP TABLE` of the referencing table — removes the object and its + two `MSysACEs` rows. + - **Its name** must differ from every other relationship's (*"There is already a relationship named '…' in + the current database."*), but may equal a table's or a query's. + + > **MSysQueries columns (8, verified).** The table has exactly: `ObjectId` (Int32, the > query object's `Id`), `Attribute` (Byte, the row kind — see below), `Flag` (Int16, attribute-specific), > `Name1` and `Name2` (Text, attribute-specific names), `Expression` (Memo, attribute-specific text — > SQL fragments), `Order` (Binary, a 4-byte big-endian per-attribute sequence counter), and `LvExtra` - > (Int32) — a long-value/overflow field that is **null in every Northwind query row** and that LibRed + > (Int32) — a long-value/overflow field observed **null** in Access-written query rows and that LibRed > leaves null (not needed for the queries it writes). Only index = composite PK `(ObjectId, Attribute, > Order)`. @@ -237,30 +246,28 @@ (`Expression`), `0x09` = a **GROUP BY** column (`Expression`; one row per group column, in order — their presence makes it a "totals" query, and the aggregate output columns are ordinary `0x06` rows, e.g. `Expression=Sum(...)`), `0x0A` = a **HAVING** predicate (`Expression`), `0x0B` = an **ORDER BY** key (`Expression`=the sort column, `Name1`=`"d"` - for **descending**, absent for ascending; one row per key, `Order` 1-based — verified against Northwind's - "Ten Most Expensive Products", `SELECT TOP 10 … ORDER BY Products.UnitPrice DESC`), `0x0C` = **complex-type + for **descending**, absent for ascending; one row per key, `Order` 1-based — verified), `0x0C` = **complex-type data** (`Flag 1` = long-text version history, `Flag 2` = MVF / attachment), `0xFF` = end. A **FROM source** (`0x05`) is either a **named table** (`Name1`=table, `Name2`=alias), a **derived table / subquery** (`Expression`=the verbatim inner subquery SQL — outer parens and `AS alias` stripped, whitespace preserved — `Name2`=alias, **no `Name1`**; - verified against Northwind's "Customer and Suppliers by City"), or, on a UNION query, one **UNION segment's + verified), or, on a UNION query, one **UNION segment's own SQL** (`Name2` = that segment's identifier). > **The `0x01` OPERATION row is the query KIND, and SELECT is one of its values** — its presence does *not* > mean the query is an action query. Access writes it on plain SELECTs as well, and omits it entirely on - > others; both shapes are common. Measured by cross-tabulating the `Flag` against DAO's `QueryDef.Type` - > over every stored query in a 19-file corpus (592 objects): + > others; both shapes are common. `Flag` values against DAO's `QueryDef.Type`: > - > | `Flag` | kind | DAO `QueryDef.Type` | corpus samples | + > | `Flag` | kind | DAO `QueryDef.Type` | status | > |---|---|---|---| - > | `1` | SELECT | `0` dbQSelect | 221 | - > | `2` | make-table (`SELECT … INTO`; target in `Name1`, external db path in `Name2`) | `80` dbQMakeTable | 3 | - > | `3` | append (`INSERT`; target table in `Name1`) | `64` dbQAppend | 27 | - > | `4` | UPDATE | `48` dbQUpdate | 19 | - > | `5` | DELETE | `32` dbQDelete | 5 | - > | `6` | crosstab (`TRANSFORM`) | `16` dbQCrosstab | 5 | - > | `7` | data definition (whole SQL in `Expression`, leading space) | `96` dbQDDL | *(none in corpus; verified by LibRed's own write/read round-trip)* | - > | `8` | pass-through | `112` dbQSQLPassThrough | *(no sample — from the published MSysQueries tables, unverified here)* | - > | `9` | UNION | `128` dbQSetOperation | 28 | + > | `1` | SELECT | `0` dbQSelect | observed | + > | `2` | make-table (`SELECT … INTO`; target in `Name1`, external db path in `Name2`) | `80` dbQMakeTable | observed | + > | `3` | append (`INSERT`; target table in `Name1`) | `64` dbQAppend | observed | + > | `4` | UPDATE | `48` dbQUpdate | observed | + > | `5` | DELETE | `32` dbQDelete | observed | + > | `6` | crosstab (`TRANSFORM`) | `16` dbQCrosstab | observed | + > | `7` | data definition (whole SQL in `Expression`, leading space) | `96` dbQDDL | *not observed in a stored query; verified by write/read round-trip only* | + > | `8` | pass-through | `112` dbQSQLPassThrough | *unverified — from the published MSysQueries tables* | + > | `9` | UNION | `128` dbQSetOperation | observed | > > On a crosstab, the `0x06` and `0x09` rows carry the extra structure in their own `Flag`: `0` = the > `TRANSFORM` value / an ordinary GROUP BY column, `1` = the `PIVOT` column heading, `2` = a row heading. @@ -283,27 +290,25 @@ > `DISTINCTROW TOP PERCENT`. `DISTINCT` and `DISTINCTROW` are separate bits and separate keywords — > `DISTINCT` dedupes output rows, `DISTINCTROW` dedupes by contributing base rows. **`Flag 9` > (`0x08|0x01`) is what Access writes for its auto-generated form/report record-source queries**, the - > `~sq_f…` / `~sq_r…` / `~sq_c…` objects, which it renders as `SELECT DISTINCTROW * FROM
` - > (measured: 107 such queries in the corpus, every one of them `DISTINCTROW`). + > `~sq_f…` / `~sq_r…` / `~sq_c…` objects, which it renders as `SELECT DISTINCTROW * FROM
`. > **A query with no `0x06` rows at all is `SELECT *`.** The absence of output columns is the encoding, not a > sign of an unreadable query — every auto-generated record-source query takes this shape. **Nested / parenthesised joins are stored flat** — one `0x05` per base table and one `0x07` per join condition, no grouping — so Access re-derives - the join tree from the conditions (verified against "Invoices": 6 tables, 5 flat joins). `Order` is a 4-byte **big-endian** + the join tree from the conditions (verified). `Order` is a 4-byte **big-endian** per-attribute counter (stored in the Binary `Order` column). MSysQueries' only index is the composite PK `(ObjectId Int32, Attribute Byte, Order Binary)`; its Binary key encodes as `0x7F` + the raw bytes + `00 00 00 00` + a length byte. > **Row order matters.** Access writes the rows in the order **type, end, parameters (`0x02`), distinct/top, - > tables (`0x05`), columns (`0x06`), joins (`0x07`), where (`0x08`), group-by (`0x09`), order-by (`0x0B`)** — *tables before columns* (verified across five - > Northwind views). Access tolerates the wrong order for a **named** table, but a **derived** table - > defines an alias the column expressions reference, so its `0x05` row must precede the `0x06` rows or + > tables (`0x05`), columns (`0x06`), joins (`0x07`), where (`0x08`), group-by (`0x09`), order-by (`0x0B`)** — + > *tables before columns* (verified). Access tolerates the wrong order for a **named** table, but a + > **derived** table defines an alias the column expressions reference, so its `0x05` row must precede the `0x06` rows or > Access opens the database yet **fails to run the view**. > > **Long `Expression` lives on an LVAL page.** `Expression` is a Memo, so a subquery longer than the > 64-byte inline limit is written to an LVAL page (§8) — required for Access to *run* the view (an - > inlined long value opens but won't execute). Verified: a LibRed derived-table UNION view returns the - > same rows in Access as the equivalent Northwind view. + > inlined long value opens but won't execute). Verified against Access. > > **CREATE PROCEDURE** is stored identically to a view (Type-5 `MSysObjects` row + `MSysQueries` rows) — > a stored query is a stored query — with one `0x02` parameter row per declared parameter. The Access @@ -311,13 +316,13 @@ > `@name`; Access stores the **bare** name (the `@` is stripped — `@Beginning_Date` → `Name1=Beginning_Date`) > while the body keeps the `@` reference verbatim: `CREATE PROCEDURE name (p1 datatype, p2 datatype) AS > select` or `CREATE PROCEDURE name p1 datatype AS select`. Verified: a LibRed-written parameterized query - > runs in Access and honours supplied parameter values. **Read-back:** LibRed reconstructs a parameterized query with a leading `PARAMETERS - > name Type, …;` clause (the `0x02` rows) and lowers body references to a declared name into engine - > parameters, so LibRed's own engine executes the stored procedure when values are supplied. + > runs in Access and honours supplied parameter values. **Read-back:** LibRed reconstructs a parameterized + > query with a leading `PARAMETERS name Type, …;` clause (the `0x02` rows) and lowers body references to a + > declared name into engine parameters, so LibRed's own engine executes the stored procedure when values are supplied. > > **Complex-column system tables (ACE 12+ only).** Access 2007 introduced multi-value and attachment > columns, and with them `MSysComplexColumns` (the registry) plus nine `MSysComplexType_*` flat storage - > tables. **Jet 4 has none of them.** Verified against a DAO-created ACE 12 database: + > tables. **Jet 4 has none of them.** As the engine creates them (verified, DAO-created ACE 12 database): > > | table | columns (id) | indexes | `MSysObjects.Flags` | > | --- | --- | --- | --- | @@ -330,8 +335,7 @@ > `Type` = 1, owner = the Engine SID. > > **`MSysComplexColumns` is load-bearing for object creation even when unused.** ACE consults it whenever - > it creates a new catalog object, and only then. Verified by dropping it from a working DAO-created - > database and exercising the surface (`AceDdlOnLibRedDatabaseProbeTest`): + > it creates a new catalog object, and only then (verified): > > | operation | without `MSysComplexColumns` | > | --- | --- | @@ -342,10 +346,9 @@ > > So it is exactly the two statements that add an `MSysObjects` row that need it — not the DDL surface as a > whole, and not a fixed system-table bind (the `CREATE VIEW` error names the table outright). It is - > **read-only** from ACE's side: a `CREATE TABLE`, a table of seven varied column types, and a - > `CREATE INDEX` all leave it at **0 rows**. Isolated against the other system tables — dropping - > `MSysComplexType_Text` changes nothing, dropping `MSysQueries` gives a different error. LibRed creates all - > ten in `DatabaseCreator.CreateEmpty` for version ≥ `0x02`, which is what lets ACE run DDL in a + > **read-only** from ACE's side: `CREATE TABLE` and `CREATE INDEX` leave it at **0 rows**. The dependency is + > on this table specifically — without `MSysComplexType_Text` the statements still succeed, and without + > `MSysQueries` they fail with a different error. LibRed creates all ten in `DatabaseCreator.CreateEmpty` for version ≥ `0x02`, which is what lets ACE run DDL in a > LibRed-created database. > **Action-query procedure bodies** (a CREATE PROCEDURE body that is not a SELECT) are stored with a @@ -369,9 +372,9 @@ (name), `szObject` (child/referencing table), `szColumn` (child column), `szReferencedObject` (parent table), `szReferencedColumn`, `icolumn` (0-based column order within the key), `ccolumn` (total column count of the key, repeated on every row), `grbit` (flags: `0x02` - don't-enforce, `0x100` cascade-update, `0x1000` cascade-delete, `0x2000` delete-set-null). Verified against Northwind: an - enforced, no-cascade single-column FK stores `ccolumn = 1`, `icolumn = 0`, `grbit = 0`; the - cascade nav-pane relationships store `grbit = 0x1100` (update+delete cascade). + don't-enforce, `0x100` cascade-update, `0x1000` cascade-delete, `0x2000` delete-set-null). Verified: an + enforced, no-cascade single-column FK stores `ccolumn = 1`, `icolumn = 0`, `grbit = 0`; a relationship + cascading both update and delete stores `grbit = 0x1100`. > **Writing a relationship.** Access records a relationship purely in `MSysRelationships` (there is > **no** `MSysObjects` row for it) **plus** a non-unique index on the child table's FK column(s) — @@ -404,9 +407,8 @@ > relationship via the table, not by dropping that index). > > **Renaming a table or column (ACE-verified).** Because this table stores its tables and columns **by - > name**, a rename has to repoint them — and ACE does. Measured (Jet suite's `RenameFanOutProbeTest`, - > against a real ACE engine via the DAO/ADOX rename path): renaming a table rewrites `szObject` / - > `szReferencedObject`, renaming a column rewrites `szColumn` / `szReferencedColumn`, and in both cases the + > name**, a rename has to repoint them — and ACE does. Verified against ACE (via the DAO/ADOX rename path): + > renaming a table rewrites `szObject` / `szReferencedObject`, renaming a column rewrites `szColumn` / `szReferencedColumn`, and in both cases the > relationship keeps its own `szRelationship` name and its enforcement — the rename is **not** refused for a > table in an enforced relationship. Nothing else moves: indexes (including the PK) keep their own names and > need no fixup because they reference the table and its columns **by id**, and a renamed column keeps its @@ -431,11 +433,10 @@ The Access **Complex** column (on-disk type `0x12`, len 4 — see [data-types](data-types.md)) implements attachment and multi-value columns. Its in-row value is a 4-byte **complex id**; the actual data lives in a -per-column backing table, wired up through hidden system tables. Decoded from the dev-edition Northwind -fixture; LibRed **reads every piece as an ordinary table but does not yet auto-resolve** a `0x12` column to -its backing rows (a `SELECT` of it returns the raw 4-byte id as `byte[]`). +per-column backing table, wired up through hidden system tables. LibRed **reads every piece as an ordinary +table but does not yet auto-resolve** a `0x12` column to its backing rows (a `SELECT` of it returns the raw 4-byte id as `byte[]`). -Three layers, all ordinary hidden/system tables: +Four layers — the user table, then three kinds of ordinary hidden/system table: 1. **The user table** holds the `Complex` column; each row's value is the 4-byte complex id. 2. **`MSysComplexColumns`** maps each complex column to its backing table: diff --git a/src/LibRed/docs/functions.md b/src/LibRed/docs/functions.md index 4db2352a5..1c5fc64a0 100644 --- a/src/LibRed/docs/functions.md +++ b/src/LibRed/docs/functions.md @@ -1,31 +1,45 @@ -# Supported functions (VBA / Access expression surface) +# Supported functions -LibRed's expression evaluator (`LibRed.Engine/Execution/ExpressionEvaluator.cs`) implements the Access / -Jet scalar function surface. This is the authoritative catalog of what's supported. +LibRed's expression evaluator (`LibRed.Engine/Execution/ExpressionEvaluator.cs`) implements the Access / Jet +scalar function surface, and on top of it a set of standard-SQL functions Access does not have. This is the +authoritative catalog of both. -**Where functions can be used.** There is **one** `ExpressionEvaluator`, shared by every place an -expression appears — so a function added for one context is available in all of them: +- [Access / VBA functions](#access--vba-functions) — the surface ACE has, verified against it. +- [Extended functions](#extended-functions) — LibRed extensions, which ACE does not have. +- [Not supported](#not-supported-by-design) — what is absent on purpose. + +--- + +## How functions work + +**Where functions can be used.** There is **one** `ExpressionEvaluator`, shared by every place an expression +appears, so a function added for one context is available in all of them: - `SELECT` projections, `WHERE`, `ORDER BY`, `GROUP BY` / `HAVING` - column `DEFAULT` expressions (see [page-02c-default-values.md](format/page-02c-default-values.md)) - table/column `CHECK` constraints The only difference between contexts is the **scope**: a query evaluates against a **row scope** (so -`IIf([N] > 10, …)` can read column `N`), while a `DEFAULT` / niladic-context evaluates against an **empty -scope** (a column reference throws "Column not found" — a default is row-blind by design). - -**Conventions.** All functions **propagate NULL** (a NULL argument yields NULL) unless noted; string -positions are **1-based**; string comparisons default to **case-insensitive** (Access "Option Compare -Database" = Text) and take an optional compare argument. Values follow VBA sign/rounding conventions -(`CInt`/`CLng`/`CByte` use banker's rounding). **Dates compare by their OLE Automation serial**, not -chronologically: below the 1899-12-30 epoch the day count goes negative while the time fraction stays -positive, so 1899-12-29 06:00 is -1.25 and 18:00 is -1.75 and ACE orders the later time first. LibRed -matches that, because `IndexKeyEncoder` writes the same serial as the index key and the two paths must -agree (see `AcePreEpochDateProbeTest`). Date *functions* are unaffected — they work in date space. -Function argument counts are validated against ACE's JES, including optional arguments and Jet quirks: -`IIf` accepts two or three arguments (an omitted false branch yields NULL), `Choose` needs an index and at -least one choice, and `Switch` needs at least one complete condition/value pair. Aggregate calls are checked -by the same contract instead of bypassing scalar validation. +`IIf([N] > 10, …)` can read column `N`), while a `DEFAULT` evaluates against an **empty scope** (a column +reference throws "Column not found" — a default is row-blind by design). The exception is the +[extended functions](#extended-functions): anything stored in the file must use only the Access / VBA ones. + +**Conventions.** + +- Every function **propagates Null** (a Null argument gives Null) unless noted. +- String positions are **1-based**. +- String comparisons are **case-insensitive** by default (Access "Option Compare Database" = Text) and take an + optional compare argument. +- Values follow VBA sign and rounding conventions (`CInt`/`CLng`/`CByte` use banker's rounding). +- **Dates compare by their OLE Automation serial**, not chronologically: below the 1899-12-30 epoch the day count + goes negative while the time fraction stays positive, so 1899-12-29 06:00 is -1.25 and 18:00 is -1.75, and ACE + orders the later time first. LibRed matches that, because `IndexKeyEncoder` writes the same serial as the index + key and the two paths must agree (see `AcePreEpochDateProbeTest`). Date *functions* are unaffected — they work + in date space. +- **Argument counts are validated** against a per-function range, including optional arguments and Jet quirks: + `IIf` accepts two or three arguments (an omitted false branch gives Null), `Choose` needs an index and at least + one choice, and `Switch` needs at least one complete condition/value pair. Aggregate calls are checked by the + same contract. A wrong count is refused rather than silently ignored. > **Two expression services — what "ACE has it" means.** Access has (1) the **Jet/ACE OLE DB Expression > Service (JES)**, the built-in set the ACE OLE DB provider carries **standalone**, and (2) the **Access @@ -33,109 +47,232 @@ by the same contract instead of bypassing scalar validation. > the **JES (standalone)** surface — the correct reference for a standalone engine. Functions that live only > in the application service (`Nz`, `Split`, `Environ`, `CurDir`, `CurrentUser`, the domain aggregates > `DCount`/`DLookup`/…) are therefore **correctly absent**, matching the OLE DB provider ("Undefined -> function"), not a gap. The whole surface below is **verified against ACE's JES** via a function-whitelist -> probe sweep (`FunctionWhitelist*Probe`); it is close but *not proven exhaustive* — re-run the sweep when in -> doubt. +> function"), not a gap. --- -## Scalar functions +## Access / VBA functions + +The JES surface, **verified against ACE** via a function-whitelist probe sweep (`FunctionWhitelist*Probe`). It is +close but *not proven exhaustive* — re-run the sweep when in doubt. + +### Type conversion + +`CBool` `CByte` `CInt` `CLng` `CSng` `CDbl` `CCur` `CStr` `CDate` `CVar` + +- A **Boolean** converts as VARIANT_BOOL, so True is **-1**, not 1: `CInt`/`CLng`/`CSng`/`CDbl`/`CCur` all give + -1, and `CByte` overflows because a byte cannot hold it. +- `CStr` renders a Double at **15 significant digits** and a Single at **7** (the OA/VB convention, not .NET's + shortest round-trippable form), and a Boolean as `"-1"` — the Jet Expression Service's answer, where the VBA + runtime proper would say `"True"`. +- `CBool` accepts a numeric string (`"-1"`) and a non-integral number. +- `CCur` rounds to 4 decimal places, and is ACE's route to a decimal. +- `CVar` is a pass-through — LibRed has no distinct Variant type. + +(Verified against ACE in `LibRed.Core.Tests.AceVbaConversionProbeTest`.) + +### Math + +`Abs` `Sgn` `Int` (floor, toward −∞) `Fix` (truncate, toward zero) `Round` (banker's) `Sqr` `Exp` `Log` +(natural) `Sin` `Cos` `Tan` `Atn` `Rnd` + +`Abs`, `Int`, `Fix` and `Round` keep their operand's type; `Int` and `Fix` of a date give a date. `Sgn` is a +Long (Int32) where ACE's is an Integer (Int16); the rest are Doubles. An argument outside a function's domain (`Sqr(-1)`, `Log(0)`) is an invalid +procedure call, and a result past a Double an overflow. -**Type conversion** — `CBool` `CByte` `CInt` `CLng` `CSng` `CDbl` `CCur` `CDec` `CStr` `CDate` `CVar` -(`CVar` is a pass-through — LibRed has no distinct Variant type; `CCur` rounds to 4 dp). A **Boolean** -converts as VARIANT_BOOL, so True is **-1**, not 1 — `CInt`/`CLng`/`CSng`/`CDbl`/`CCur` all yield -1, and -`CByte` overflows because a byte cannot hold it. `CStr` renders a Double at **15 significant digits** and a -Single at **7** (the OA/VB convention, not .NET's shortest round-trippable form), and a Boolean as `"-1"` — -note that is the Jet Expression Service's answer, where the VBA runtime proper would say `"True"`. `CBool` -accepts a numeric string (`"-1"`) and a non-integral number. `CDec` has **no ACE equivalent** — the -expression service has no such function — so it is a LibRed extension with no parity contract; `CCur` is -ACE's route to a decimal. (All verified against ACE in -`LibRed.Core.Tests.AceVbaConversionProbeTest`.) +### String -**Math** — `Abs` `Sgn` `Int` (floor, toward −∞) `Fix` (truncate, toward zero) `Round` (banker's) `Sqr` -`Exp` `Log` (natural) `Sin` `Cos` `Tan` `Atn` `Rnd` `Timer`. +`Len` `LCase` `UCase` `Trim` `LTrim` `RTrim` `Left` `Right` `Mid` `InStr` `InStrRev` `Replace` `Space` `String` +`StrReverse` `StrComp` `StrConv` `Str` `Val` `Chr` `Asc` `Hex` `Oct` -**String** — `Len` `LCase` `UCase` `Trim` `LTrim` `RTrim` `Left` `Right` `Mid` `InStr` `InStrRev` `Replace` -`Space` `String` `StrReverse` `StrComp` `StrConv` `Str` `Val` `Chr` `Asc` `Hex` `Oct`. +- `Trim`/`LTrim`/`RTrim` strip the space and the ideographic space U+3000 — nothing else. +- `Asc` and `Chr` work in the system ANSI code page (`Chr` takes 0–255). +- A value that is not text is read as `CStr` writes it. -**Formatting** — `Format` (VBA→.NET custom + named formats; culture-driven, so date/currency named formats -are locale-dependent by design) · `FormatCurrency` `FormatNumber` `FormatPercent` `FormatDateTime` · -`Partition` (a `"lower:upper"` range-bucket label). +### Date and time -**Date / time** — `Now` `Date` `Time` · `DateAdd` `DateDiff` `DatePart` `DateSerial` `TimeSerial` -`DateValue` `TimeValue` · `Year` `Month` `Day` `Hour` `Minute` `Second` `Weekday` (Sunday = 1) · `MonthName` -`WeekdayName` · `IsDate`. (`Time`/`TimeValue` sit on the Jet epoch 1899-12-30.) +- **Current** — `Now` `Date` `Time` `Timer` +- **Arithmetic** — `DateAdd` `DateDiff` `DatePart` `DateSerial` `TimeSerial` +- **Parsing** — `DateValue` `TimeValue` +- **Parts** — `Year` `Month` `Day` `Hour` `Minute` `Second` `Weekday` (Sunday = 1) +- **Names** — `MonthName` `WeekdayName` -**Logical / selection** — `IIf` `Choose` (1-based; out-of-range → NULL) `Switch` (first true condition's -value; even arg count required). +`Time` and `TimeValue` sit on the Jet epoch 1899-12-30. LibRed keeps milliseconds where ACE rounds to the second. -**Inspection / predicates** — `IsNull` `IsNumeric` `IsDate` `IsError` (always False — LibRed has no error -value type) `TypeName` `VarType`. +### Formatting -**Financial** (closed-form annuity/depreciation; `Rate` by Newton–Raphson; verified vs ACE to ~1e-6) — -`Pmt` `FV` `PV` `NPer` `IPmt` `PPmt` `Rate` `SLN` `SYD` `DDB`. (`IRR`/`NPV` need an array argument — no -scalar-SQL form.) +`Format` `FormatCurrency` `FormatNumber` `FormatPercent` `FormatDateTime` `Partition` -**Colour** — `RGB` (`r + g·256 + b·65536`) · `QBColor` (16-entry BGR table). +`Format` maps VBA custom and named formats onto .NET; it is culture-driven, so the date and currency named formats +are locale-dependent by design. `Partition` gives a `"lower:upper"` range-bucket label. + +### Logical and selection + +- `IIf` — only the branch taken is evaluated. +- `Choose` — 1-based; an index out of range gives Null. +- `Switch` — the first true condition's value; the arguments come in pairs. + +### Inspection + +`IsNull` `IsNumeric` `IsDate` `IsError` `TypeName` `VarType` + +`IsError` is always False — LibRed has no error value type — but its argument is still evaluated, so an error in +it is raised. + +### Financial + +`Pmt` `FV` `PV` `NPer` `IPmt` `PPmt` `Rate` `SLN` `SYD` `DDB` + +Closed-form annuity and depreciation, and `Rate` by Newton–Raphson; verified against ACE to ~1e-6. `IRR`/`NPV` need +an array argument, so have no scalar-SQL form. + +### Colour + +`RGB` (`r + g·256 + b·65536`) · `QBColor` (the 16-entry BGR table). ### Name variants (`$` / `B` / `W`) -- **`$` — string-returning** (`Left$`, `UCase$`, `Chr$`, `Str$`, `Format$`, …): the lexer allows a trailing - `$` on an identifier and the evaluator strips it before dispatch, so **every** base function gains its `$` - form and computes the same value. (ACE exposes `$` only for classic functions; the alias is harmless where - ACE doesn't.) +- **`$` — string-returning** (`Left$`, `UCase$`, `Chr$`, `Str$`, `Format$`, …): the lexer allows a trailing `$` on + an identifier and the evaluator strips it before dispatch, so **every** function gains its `$` form and computes + the same value. (ACE exposes `$` only for classic functions; the alias is harmless where ACE doesn't.) - **`B` — byte-based** on the UTF-16 layout (2 bytes/char): `AscB` `LenB` `LeftB` `RightB` `MidB` `InStrB` (`LenB('abc')` = 6). `ChrB` is intentionally **absent** (ACE's JES has none either). -- **`W` — wide / Unicode code point**: `AscW` (first char's code point) · `ChrW` (char for a code point, not - limited to a byte — `ChrW(233)` = 'é'). +- **`W` — wide / Unicode**: `AscW` (the first character's UTF-16 code unit, signed) · `ChrW` (the character for a + code unit, -32768 to 65535 — `ChrW(233)` = 'é'). ### Niladic (callable without parentheses) Only **`Now`** is niladic — ACE accepts bare `Now` (e.g. `DATETIME DEFAULT Now`). Bare `Date` / `Time` are -**reserved type keywords** in Jet SQL and must be written `Date()` / `Time()` (parsed as calls). A real -column named `Now` still shadows the function. +**reserved type keywords** in Jet SQL and must be written `Date()` / `Time()`. A real column named `Now` still +shadows the function. ### Default-only generators -These are **not** callable in a `SELECT` (ACE errors "Undefined function") but are valid as a column -`DEFAULT`, evaluated per inserted row: +These are **not** callable in an ACE `SELECT` ("Undefined function") but are valid as a column `DEFAULT`, evaluated +per inserted row: -- **`GenUniqueID()`** — a random signed `Long` (Int32); valid only on a `LONG` column. It is the mechanism - behind a **"Random" AutoNumber** (see [system-catalog](format/system-catalog.md)). +- **`GenUniqueID()`** — a random signed `Long` (Int32); valid only on a `LONG` column. It is the mechanism behind a + **"Random" AutoNumber** (see [system-catalog](format/system-catalog.md)). - **`GenGUID()`** — a fresh `Guid` per row; EF Core models it as `HasDefaultValueSql("GenGUID()")` for store-generated GUID keys. -(LibRed is slightly more permissive than ACE here — it *will* also evaluate `GenUniqueID()` / `CDec()` in a -`SELECT`, which ACE rejects. Harmless: it never produces a file ACE can't read.) - ---- +LibRed is slightly more permissive than ACE here — it *will* also evaluate them in a `SELECT`. Harmless: it never +produces a file ACE can't read. -## System variables +### System variables Connection-scoped, written `@@NAME`, usable anywhere an expression is allowed: -- **`@@IDENTITY`** — the last AutoNumber generated on this connection (NULL if none). Only overwritten by an - insert that actually generates one, so an intervening keyless insert leaves it intact — which is why EF - Core reads it in the `SELECT` immediately after an `INSERT`. +- **`@@IDENTITY`** — the last AutoNumber generated on this connection (Null if none). Only overwritten by an insert + that actually generates one, so an intervening keyless insert leaves it intact — which is why EF Core reads it + in the `SELECT` immediately after an `INSERT`. - **`@@ROWCOUNT`** — rows affected by the previous statement. -A bare `SELECT @@IDENTITY` / `SELECT @@ROWCOUNT` (a comma list of system vars, optionally aliased) is -allowed without a `FROM` — the one all-`@@var` projection that may omit it, matching ACE. +A bare `SELECT @@IDENTITY` / `SELECT @@ROWCOUNT` (a comma list of system variables, optionally aliased) is allowed +without a `FROM` — the one all-`@@var` projection that may omit it, matching ACE. + +### Aggregates + +`Count` · `Sum` · `Avg` · `Min` · `Max` · `First` / `Last` · `StDev` / `Var` · `StDevP` / `VarP` + +- Each takes `DISTINCT`. +- `First` / `Last` are the first/last row's value in scan order — **not** Null-filtered. +- `StDev` / `Var` are the sample statistics (÷ n−1, Null for n < 2); `StDevP` / `VarP` the population ones (÷ n). + `StdDev` / `StdDevP` are accepted spellings. + +**Result types** (matching Access and LINQ, so the EF provider round-trips without a cast): `Sum` **keeps the input +type** (int→int, long→long, decimal→decimal); `Avg` is a Double unless the input is Currency/Decimal; `Min`/`Max` +keep the column's own value and type. `Sum`/`Avg`/`Min`/`Max` of no rows is Null; `Count` is 0. --- -## Aggregate functions +## Extended functions + +LibRed extensions. ACE has none of these, so there is no parity contract to honour; where a standard exists they +follow it, and otherwise SQL Server or PostgreSQL. The engine accepts them in any query, whichever SQL mode EF is +using; SQL written to run on ACE as well must leave them out. + +> **Queries only — never in anything stored in the file.** Extended functions are not allowed in views, procedures +> (saved queries), `CHECK` constraints, column `DEFAULT`s or calculated columns. Some of those may work while the +> file is only ever opened by LibRed, but the expression is stored in the file, and ACE or Access opening it will +> not understand it. + +### Type conversion + +- **`CLngLng`** — VBA's LongLong conversion (the JES reports it undefined). Reads its argument as `CLng` does, into + an **Int64**, reading text exactly rather than through a Double. +- **`CDec`** — to a Decimal. The JES has no such function; `CCur` is ACE's route to a decimal. + +### Math + +`Floor` `Ceiling`/`Ceil` `Sign` `Sqrt` `Ln` `Log10` `Log(base, x)` `Power(x, y)` `Asin` `Acos` `Atan` +`Atan2(y, x)` `Sinh` `Cosh` `Tanh` `Degrees` `Radians` `Pi()` + +- Where Access has the function under another name it *is* that function, reading its arguments and failing the + same way: `Floor` is `Int`, `Sqrt` is `Sqr`, `Ln` is `Log`, `Atan` is `Atn`, `Sign` is `Sgn`, and `Power` is the + `^` operator. +- `Floor` and `Ceiling` keep their operand's type; `Sign` is a Long, as `Sgn` is; the rest are Doubles. +- `Log` with one argument is still Access's natural log; the two-argument form takes the base first, as the + standard and PostgreSQL do (SQL Server's `LOG(x, base)` is the other way round). +- An argument outside a function's domain (`Asin(2)`, `Log(1, 5)`) is an invalid procedure call, and a result past + a Double an overflow. + +### Null handling and selection + +- **`Coalesce(x, …)`** — the first non-Null argument. One argument or more. +- **`NullIf(x, y)`** — Null when the two are equal, otherwise `x`; typed as `x`. +- **`Greatest(x, …)` / `Least(x, …)`** — the largest / smallest argument, ignoring Null arguments as SQL Server + and PostgreSQL do. Extended mode translates `Math.Max`/`Math.Min` to them. + +`Coalesce`, `Greatest` and `Least` declare the type their arguments unify to, as `CASE` does (`CASE` itself is +syntax, not a function). + +### Date and time + +- **`DatePart`** also takes `"ms"`, `"mcs"` and `"ns"`: the millisecond, microsecond and nanosecond of the time. +- **`DateAdd`** also takes `"ms"`. +- **`DateDiff`** also takes `"ms"`, counting in **Int64** — a millisecond difference passes Int32 after 25 days. + +ACE's interval list stops at `"s"`. + +### Storage size + +**`DataLength(x)`**, SQL Server's — the bytes a value takes as Access stores it. Text is 2 per character (UTF-16, +trailing spaces counted, on-disk compression ignored), binary its length, Byte 1, Integer 2, Long/Single 4, +Double/Currency/Date/BIGINT 8, GUID 16, Decimal 17; a Boolean — a bit of the null bitmap on disk — counts 1, as SQL +Server counts a `bit`. A Long. + +### Aggregates + +- **Standard statistic names** — `STDDEV_SAMP`, `STDDEV_POP`, `VAR_SAMP` and `VAR_POP`: the same aggregates as + `StDev`, `StDevP`, `Var` and `VarP`. +- **Binary set functions** over `(y, x)` pairs, using only the pairs where neither is Null — `CORR`, `COVAR_POP`, + `COVAR_SAMP`, `REGR_COUNT`, `REGR_AVGX`, `REGR_AVGY`, `REGR_SXX`, `REGR_SYY`, `REGR_SXY`, `REGR_SLOPE`, + `REGR_INTERCEPT` and `REGR_R2`. `REGR_COUNT` is a Long, the rest Doubles; they take no `DISTINCT`. Every x the + same makes the slope, intercept, R² and correlation Null; every y the same makes the correlation Null and R² 1. +- **Ordered-set aggregates** — `PERCENTILE_CONT(p)` and `PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY x [DESC])`: the + value at fraction `p` (0 to 1) of the ordered non-Null values. `PERCENTILE_CONT` interpolates linearly (a + Double, or a date for dates); `PERCENTILE_DISC` is the first value whose cumulative share reaches `p`, in the + key's own type. Neither takes `DISTINCT`. +- **`LISTAGG([DISTINCT] x [, 'separator']) WITHIN GROUP (ORDER BY k [DESC], …)`** — the non-Null values as text + (each written as `&` writes it) in that order, joined by the separator: a string literal, as the standard has + it, and none when left out. Null when there are no values. +- **`FILTER (WHERE condition)`** — every aggregate, Access's included, takes it after the call (and after `WITHIN + GROUP`): only the rows the condition is true for go in, so `COUNT(*) FILTER (WHERE x > 1)` counts those rows, + and a group none of whose rows pass has what an empty group has. + +### Window functions -The complete Access SQL aggregate set (`QueryPlanner` + `QueryExecutor`), each supporting `DISTINCT`: +Called with `OVER ([PARTITION BY …] [ORDER BY …] [frame])`: -`Count` · `Sum` · `Avg` · `Min` · `Max` · `First` / `Last` (first/last row's value in scan order — **not** -null-filtered) · `StDev` / `Var` (sample: ÷ n−1, NULL for n < 2) · `StDevP` / `VarP` (population: ÷ n). -`StdDev` / `StdDevP` are accepted spellings. +- **Ranking** — `ROW_NUMBER()` `RANK()` `DENSE_RANK()` `NTILE(n)` `PERCENT_RANK()` `CUME_DIST()` +- **Offset** — `LAG(x [, offset [, default]])` `LEAD(x [, offset [, default]])` +- **Value** — `FIRST_VALUE(x)` `LAST_VALUE(x)` `NTH_VALUE(x, n) [FROM FIRST | FROM LAST]` +- **Aggregates** — every aggregate above, Access's and the extended ones, over a window. The one-argument + aggregates also take `DISTINCT` there. -**Result-type contract** (matches Access + LINQ so the EF provider round-trips without a cast): `Sum` -**preserves the input type** (int→int, long→long, decimal→decimal); `Avg` is `Double` unless the input is -Currency/Decimal; `Min`/`Max` keep the column's own value and type; `Sum`/`Avg`/`Min`/`Max` of no rows is -NULL (`Count` returns 0). +`LAG`, `LEAD` and the value functions take `RESPECT NULLS` or `IGNORE NULLS` before the `OVER`. The frame and the +behaviour over a grouped query are described in the [README](../README.md). --- @@ -146,5 +283,5 @@ NULL (`Count` returns 0). representation). - **No scalar-SQL form:** `IRR` / `NPV` (array argument); `Array` / `Join` / `CVErr` (VBA-only). -See [page-02c-default-values.md](format/page-02c-default-values.md) for how these functions behave specifically in a column -`DEFAULT` (the DDL-parser-vs-expression-service split, and the forbidden categories). +See [page-02c-default-values.md](format/page-02c-default-values.md) for how these functions behave specifically in a +column `DEFAULT` (the DDL-parser-vs-expression-service split, and the forbidden categories). diff --git a/src/EFCore.Jet.Data/JetDecimalConverter.cs b/src/Shared/Data/JetDecimalConverter.cs similarity index 90% rename from src/EFCore.Jet.Data/JetDecimalConverter.cs rename to src/Shared/Data/JetDecimalConverter.cs index 04b6f69fa..be86c93ce 100644 --- a/src/EFCore.Jet.Data/JetDecimalConverter.cs +++ b/src/Shared/Data/JetDecimalConverter.cs @@ -34,9 +34,28 @@ namespace EntityFrameworkCore.Jet.Data /// divergence and sweeps 20,000 values to confirm the two never differ by more than one unit in the /// 15th significant digit. /// + /// + /// The source is shared: it is compiled into both EntityFrameworkCore.Jet.Data and LibRed.Core, which + /// reference neither each other nor a common assembly that could hold it. Every LibRed assembly uses the + /// LibRed.Core copy. Any conversion that may meet a or goes + /// through here — in place of Convert.ToDecimal, never a cast. + /// /// public static class JetDecimalConverter { + /// + /// , except that a or + /// is converted at its real precision (, + /// ). + /// + public static decimal ToDecimal(object? value, IFormatProvider? provider = null) + => value switch + { + double d => FromDouble(d), + float f => FromSingle(f), + _ => System.Convert.ToDecimal(value, provider), + }; + private const int DecScaleMax = 28; private const int ScaleShift = 16; diff --git a/test/EFCore.Jet.Data.Tests/ConnectionPoolingTest.cs b/test/EFCore.Jet.Data.Tests/ConnectionPoolingTest.cs index bba6fa22d..f5f073244 100644 --- a/test/EFCore.Jet.Data.Tests/ConnectionPoolingTest.cs +++ b/test/EFCore.Jet.Data.Tests/ConnectionPoolingTest.cs @@ -1,5 +1,6 @@ using System; using System.Data.OleDb; +using System.Reflection; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace EntityFrameworkCore.Jet.Data.Tests @@ -378,6 +379,40 @@ public void GetSchema_From_Closed_Connection() Assert.Throws(() => connection.GetSchema()); } + [TestMethod] + public void Pooled_Inner_Connection_Is_Reused_After_Close() + { + // Open() rebuilds the connection string (here, the "Jet OLEDB:" key comes back lower-cased), so the + // pool must be keyed by the rebuilt string on both sides. Otherwise every Open() creates a new native + // connection while the pool keeps all the old ones open, and Jet 4.0 fails once 64 are held. + var innerConnectionProperty = typeof(JetConnection).GetProperty( + "InnerConnection", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)!; + var connectionString = JetConnection.GetConnectionString(StoreName, Helpers.DataAccessProviderFactory) + + ";Jet OLEDB:Database Locking Mode=1"; + + JetConfiguration.UseConnectionPooling = true; + try + { + using var connection = new JetConnection(connectionString, Helpers.DataAccessProviderFactory); + + connection.Open(); + var firstInnerConnection = innerConnectionProperty.GetValue(connection); + connection.Close(); + + connection.Open(); + var secondInnerConnection = innerConnectionProperty.GetValue(connection); + connection.Close(); + + Assert.AreSame(firstInnerConnection, secondInnerConnection); + } + finally + { + JetConfiguration.UseConnectionPooling = false; + JetConnection.ClearAllPools(); + } + } + [TestMethod] public void Transaction_Execute_Close_Open_Execute() { diff --git a/test/EFCore.Jet.FunctionalTests/Query/Translations/MathTranslationsJetTest.cs b/test/EFCore.Jet.FunctionalTests/Query/Translations/MathTranslationsJetTest.cs index 3abb557aa..1b8cc6b36 100644 --- a/test/EFCore.Jet.FunctionalTests/Query/Translations/MathTranslationsJetTest.cs +++ b/test/EFCore.Jet.FunctionalTests/Query/Translations/MathTranslationsJetTest.cs @@ -73,7 +73,7 @@ public override async Task Ceiling() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE IIF(FIX(`b`.`Double`) = `b`.`Double`, FIX(`b`.`Double`), FIX(`b`.`Double`) + 1.0) = 9.0 +WHERE -INT(-`b`.`Double`) = 9.0 """); } @@ -85,7 +85,7 @@ public override async Task Ceiling_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE IIF(FIX(`b`.`Float`) = `b`.`Float`, FIX(`b`.`Float`), FIX(`b`.`Float`) + 1) = 9 +WHERE -INT(-`b`.`Float`) = 9 """); } @@ -97,7 +97,7 @@ public override async Task Floor_decimal() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Decimal`) = 8.0 +WHERE INT(`b`.`Decimal`) = 8.0 """); } @@ -109,7 +109,7 @@ public override async Task Floor_double() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Double`) = 8.0 +WHERE INT(`b`.`Double`) = 8.0 """); } @@ -121,7 +121,7 @@ public override async Task Floor_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Float`) = 8 +WHERE INT(`b`.`Float`) = 8 """); } @@ -665,7 +665,7 @@ public override async Task Acos() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (1.5707963267948966 + ATN(-`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0))) > 1.0 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (1.5707963267948966 + (ATN(-`b`.`Double` / (SQR(-(`b`.`Double` * `b`.`Double`) + 1.0) + 1.0)) * 2.0)) > 1.0 """); } @@ -677,7 +677,7 @@ public override async Task Acos_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND (1.5707963267948966 + ATN(-`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1))) > 0.0 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND (1.5707963267948966 + (ATN(-`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2.0)) > 0.0 """); } @@ -692,7 +692,7 @@ public override async Task Asin() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND ATN(`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0)) > -1.7976931348623157E+308 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (ATN(`b`.`Double` / (SQR(-(`b`.`Double` * `b`.`Double`) + 1.0) + 1.0)) * 2.0) > -1.7976931348623157E+308 """); } @@ -704,7 +704,7 @@ public override async Task Asin_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND IIF(ATN(`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1)) IS NULL, NULL, CDBL(ATN(`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1)))) > -1.7976931348623157E+308 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND IIF((ATN(`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2) IS NULL, NULL, CDBL(ATN(`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2)) > -1.7976931348623157E+308 """); } @@ -746,7 +746,7 @@ public override async Task Atan2() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Double` / 1.0) > 0.0 +WHERE IIF(1.0 = 0.0, SGN(`b`.`Double`) * 1.5707963267948966, ATN(`b`.`Double` / 1.0) + IIF(1.0 < 0.0, IIF(`b`.`Double` < 0.0, -3.141592653589793, 3.141592653589793), 0.0)) > 0.0 """); } @@ -758,7 +758,7 @@ public override async Task Atan2_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Float` / 1) > 0 +WHERE IIF(1 = 0, SGN(`b`.`Float`) * 1.5707964, ATN(`b`.`Float` / 1) + IIF(1 < 0, IIF(`b`.`Float` < 0, -3.1415927, 3.1415927), 0)) > 0 """); } diff --git a/test/EFCore.LibRed.Extended.FunctionalTests/Query/ComplexNavigationsQueryLibRedTest.cs b/test/EFCore.LibRed.Extended.FunctionalTests/Query/ComplexNavigationsQueryLibRedTest.cs index 8e2dfe997..d5e06af4c 100644 --- a/test/EFCore.LibRed.Extended.FunctionalTests/Query/ComplexNavigationsQueryLibRedTest.cs +++ b/test/EFCore.LibRed.Extended.FunctionalTests/Query/ComplexNavigationsQueryLibRedTest.cs @@ -3118,10 +3118,7 @@ public override async Task Nav_rewrite_doesnt_apply_null_protection_for_function AssertSql( """ -SELECT CASE - WHEN `l0`.`Level1_Required_Id` > 7 THEN `l0`.`Level1_Required_Id` - ELSE 7 -END +SELECT GREATEST(`l0`.`Level1_Required_Id`, 7) FROM `LevelOne` AS `l` LEFT JOIN `LevelTwo` AS `l0` ON `l`.`Id` = `l0`.`OneToOne_Optional_PK_Inverse2Id` WHERE `l0`.`Id` IS NOT NULL diff --git a/test/EFCore.LibRed.Extended.FunctionalTests/Query/DbFunctionsLibRedTest.cs b/test/EFCore.LibRed.Extended.FunctionalTests/Query/DbFunctionsLibRedTest.cs index d07effc3e..9eaaa1cbc 100644 --- a/test/EFCore.LibRed.Extended.FunctionalTests/Query/DbFunctionsLibRedTest.cs +++ b/test/EFCore.LibRed.Extended.FunctionalTests/Query/DbFunctionsLibRedTest.cs @@ -150,10 +150,7 @@ public override async Task Least(bool async) """ SELECT `o`.`OrderID`, `o`.`ProductID`, `o`.`Discount`, `o`.`Quantity`, `o`.`UnitPrice` FROM `Order Details` AS `o` -WHERE CASE - WHEN `o`.`OrderID` < 10251 THEN `o`.`OrderID` - ELSE 10251 -END = 10251 +WHERE LEAST(`o`.`OrderID`, 10251) = 10251 """); } @@ -165,10 +162,7 @@ public override async Task Greatest(bool async) """ SELECT `o`.`OrderID`, `o`.`ProductID`, `o`.`Discount`, `o`.`Quantity`, `o`.`UnitPrice` FROM `Order Details` AS `o` -WHERE CASE - WHEN `o`.`OrderID` > 10251 THEN `o`.`OrderID` - ELSE 10251 -END = 10251 +WHERE GREATEST(`o`.`OrderID`, 10251) = 10251 """); } @@ -180,10 +174,7 @@ public override async Task Least_with_nullable_value_type(bool async) """ SELECT `o`.`OrderID`, `o`.`ProductID`, `o`.`Discount`, `o`.`Quantity`, `o`.`UnitPrice` FROM `Order Details` AS `o` -WHERE CASE - WHEN `o`.`OrderID` < 10251 THEN `o`.`OrderID` - ELSE 10251 -END = 10251 +WHERE LEAST(`o`.`OrderID`, 10251) = 10251 """); } @@ -195,10 +186,7 @@ public override async Task Greatest_with_nullable_value_type(bool async) """ SELECT `o`.`OrderID`, `o`.`ProductID`, `o`.`Discount`, `o`.`Quantity`, `o`.`UnitPrice` FROM `Order Details` AS `o` -WHERE CASE - WHEN `o`.`OrderID` > 10251 THEN `o`.`OrderID` - ELSE 10251 -END = 10251 +WHERE GREATEST(`o`.`OrderID`, 10251) = 10251 """); } diff --git a/test/EFCore.LibRed.Extended.FunctionalTests/Query/NorthwindFunctionsQueryLibRedTest.Functions.cs b/test/EFCore.LibRed.Extended.FunctionalTests/Query/NorthwindFunctionsQueryLibRedTest.Functions.cs index 621e3cc97..8dc0de858 100644 --- a/test/EFCore.LibRed.Extended.FunctionalTests/Query/NorthwindFunctionsQueryLibRedTest.Functions.cs +++ b/test/EFCore.LibRed.Extended.FunctionalTests/Query/NorthwindFunctionsQueryLibRedTest.Functions.cs @@ -111,7 +111,7 @@ public override async Task Where_functions_nested(bool isAsync) """ SELECT `c`.`CustomerID`, `c`.`Address`, `c`.`City`, `c`.`CompanyName`, `c`.`ContactName`, `c`.`ContactTitle`, `c`.`Country`, `c`.`Fax`, `c`.`Phone`, `c`.`PostalCode`, `c`.`Region` FROM `Customers` AS `c` -WHERE CDBL(LEN(`c`.`CustomerID`))^2.0 = 25.0 +WHERE POWER(CDBL(LEN(`c`.`CustomerID`)), 2.0) = 25.0 """); } diff --git a/test/EFCore.LibRed.Extended.FunctionalTests/Query/PrimitiveCollectionsQueryLibRedTest.cs b/test/EFCore.LibRed.Extended.FunctionalTests/Query/PrimitiveCollectionsQueryLibRedTest.cs index e38cd3b96..9c984cf47 100644 --- a/test/EFCore.LibRed.Extended.FunctionalTests/Query/PrimitiveCollectionsQueryLibRedTest.cs +++ b/test/EFCore.LibRed.Extended.FunctionalTests/Query/PrimitiveCollectionsQueryLibRedTest.cs @@ -257,9 +257,7 @@ public override async Task Inline_collection_Min_with_two_values() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE LEAST(30, `p`.`Int`) = 30 """); } @@ -271,9 +269,7 @@ public override async Task Inline_collection_List_Min_with_two_values() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE LEAST(30, `p`.`Int`) = 30 """); } @@ -285,9 +281,7 @@ public override async Task Inline_collection_Max_with_two_values() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE GREATEST(30, `p`.`Int`) = 30 """); } @@ -299,9 +293,7 @@ public override async Task Inline_collection_List_Max_with_two_values() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE GREATEST(30, `p`.`Int`) = 30 """); } @@ -315,9 +307,7 @@ public override async Task Inline_collection_Min_with_three_values() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 25 +WHERE LEAST(30, `p`.`Int`, @i) = 25 """); } @@ -331,9 +321,7 @@ public override async Task Inline_collection_List_Min_with_three_values() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 25 +WHERE LEAST(30, `p`.`Int`, @i) = 25 """); } @@ -347,9 +335,7 @@ public override async Task Inline_collection_Max_with_three_values() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 35 +WHERE GREATEST(30, `p`.`Int`, @i) = 35 """); } @@ -363,9 +349,7 @@ public override async Task Inline_collection_List_Max_with_three_values() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 35 +WHERE GREATEST(30, `p`.`Int`, @i) = 35 """); } @@ -379,9 +363,7 @@ public override async Task Inline_collection_of_nullable_value_type_Min() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 25 +WHERE LEAST(30, `p`.`Int`, @i) = 25 """); } @@ -395,9 +377,7 @@ public override async Task Inline_collection_of_nullable_value_type_Max() SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`), (@i)) AS `v`) = 35 +WHERE GREATEST(30, `p`.`Int`, @i) = 35 """); } @@ -409,9 +389,7 @@ public override async Task Inline_collection_of_nullable_value_type_with_null_Mi """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`NullableInt`), (NULL)) AS `v`) = 30 +WHERE LEAST(30, `p`.`NullableInt`, NULL) = 30 """); } @@ -423,9 +401,7 @@ public override async Task Inline_collection_of_nullable_value_type_with_null_Ma """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`NullableInt`), (NULL)) AS `v`) = 30 +WHERE GREATEST(30, `p`.`NullableInt`, NULL) = 30 """); } @@ -1581,9 +1557,7 @@ public override async Task Min_on_MemoryExtensions() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MIN(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE LEAST(30, `p`.`Int`) = 30 """); } @@ -1595,9 +1569,7 @@ public override async Task Max_on_MemoryExtensions() """ SELECT `p`.`Id`, `p`.`Bool`, `p`.`Bools`, `p`.`DateTime`, `p`.`DateTimes`, `p`.`Enum`, `p`.`Enums`, `p`.`Int`, `p`.`Ints`, `p`.`NullableInt`, `p`.`NullableInts`, `p`.`NullableString`, `p`.`NullableStrings`, `p`.`NullableWrappedId`, `p`.`NullableWrappedIdWithNullableComparer`, `p`.`String`, `p`.`Strings`, `p`.`WrappedId` FROM `PrimitiveCollectionsEntity` AS `p` -WHERE ( - SELECT MAX(`v`.`Value`) - FROM (SELECT CLNG(30) AS `Value` UNION ALL VALUES (`p`.`Int`)) AS `v`) = 30 +WHERE GREATEST(30, `p`.`Int`) = 30 """); } diff --git a/test/EFCore.LibRed.Extended.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs b/test/EFCore.LibRed.Extended.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs index c2cf0ba36..1b35c55ce 100644 --- a/test/EFCore.LibRed.Extended.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs +++ b/test/EFCore.LibRed.Extended.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs @@ -73,10 +73,7 @@ public override async Task Ceiling() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN FIX(`b`.`Double`) = `b`.`Double` THEN FIX(`b`.`Double`) - ELSE FIX(`b`.`Double`) + 1.0 -END = 9.0 +WHERE CEILING(`b`.`Double`) = 9.0 """); } @@ -88,10 +85,7 @@ public override async Task Ceiling_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN FIX(`b`.`Float`) = `b`.`Float` THEN FIX(`b`.`Float`) - ELSE FIX(`b`.`Float`) + 1 -END = 9 +WHERE CEILING(`b`.`Float`) = 9 """); } @@ -103,7 +97,7 @@ public override async Task Floor_decimal() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Decimal`) = 8.0 +WHERE FLOOR(`b`.`Decimal`) = 8.0 """); } @@ -115,7 +109,7 @@ public override async Task Floor_double() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Double`) = 8.0 +WHERE FLOOR(`b`.`Double`) = 8.0 """); } @@ -127,7 +121,7 @@ public override async Task Floor_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Float`) = 8 +WHERE FLOOR(`b`.`Float`) = 8 """); } @@ -139,7 +133,7 @@ public override async Task Power() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CDBL(`b`.`Int`)^2.0 = 64.0 +WHERE POWER(CDBL(`b`.`Int`), 2.0) = 64.0 """); } @@ -151,7 +145,7 @@ public override async Task Power_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float`^2 > 73 AND `b`.`Float`^2 < 74 +WHERE POWER(`b`.`Float`, 2) > 73 AND POWER(`b`.`Float`, 2) < 74 """); } @@ -373,7 +367,7 @@ public override async Task Log() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` > 0.0 AND LOG(`b`.`Double`) <> 0.0 +WHERE `b`.`Double` > 0.0 AND LN(`b`.`Double`) <> 0.0 """); } @@ -385,7 +379,7 @@ public override async Task Log_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` > 0 AND LOG(`b`.`Float`) <> 0 +WHERE `b`.`Float` > 0 AND LN(`b`.`Float`) <> 0 """); } @@ -397,7 +391,7 @@ public override async Task Log_with_newBase() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` > 0.0 AND (LOG(`b`.`Double`) / LOG(7.0)) <> 0.0 +WHERE `b`.`Double` > 0.0 AND LOG(7.0, `b`.`Double`) <> 0.0 """); } @@ -409,7 +403,7 @@ public override async Task Log_with_newBase_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` > 0 AND (LOG(`b`.`Float`) / LOG(7)) <> 0 +WHERE `b`.`Float` > 0 AND LOG(7, `b`.`Float`) <> 0 """); } @@ -421,7 +415,7 @@ public override async Task Log10() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` > 0.0 AND (LOG(`b`.`Double`) / 2.302585092994046) <> 0.0 +WHERE `b`.`Double` > 0.0 AND LOG10(`b`.`Double`) <> 0.0 """); } @@ -433,7 +427,7 @@ public override async Task Log10_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` > 0 AND (LOG(`b`.`Float`) / 2.3025851) <> 0 +WHERE `b`.`Float` > 0 AND LOG10(`b`.`Float`) <> 0 """); } @@ -448,7 +442,7 @@ public override async Task Sqrt() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` > 0.0 AND SQR(`b`.`Double`) > 0.0 +WHERE `b`.`Double` > 0.0 AND SQRT(`b`.`Double`) > 0.0 """); } @@ -460,7 +454,7 @@ public override async Task Sqrt_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` > 0 AND SQR(`b`.`Float`) > 0 +WHERE `b`.`Float` > 0 AND SQRT(`b`.`Float`) > 0 """); } @@ -472,11 +466,11 @@ public override async Task Sign() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE SGN(`b`.`Double`) > 0 +WHERE SIGN(`b`.`Double`) > 0 """, // """ -SELECT SGN(`b`.`Double`) +SELECT SIGN(`b`.`Double`) FROM `BasicTypesEntities` AS `b` """); } @@ -489,11 +483,11 @@ public override async Task Sign_decimal() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE SGN(`b`.`Decimal`) > 0 +WHERE SIGN(`b`.`Decimal`) > 0 """, // """ -SELECT SGN(`b`.`Decimal`) +SELECT SIGN(`b`.`Decimal`) FROM `BasicTypesEntities` AS `b` """); } @@ -506,11 +500,11 @@ public override async Task Sign_int() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE SGN(`b`.`Int`) > 0 +WHERE SIGN(`b`.`Int`) > 0 """, // """ -SELECT SGN(`b`.`Int`) +SELECT SIGN(`b`.`Int`) FROM `BasicTypesEntities` AS `b` """); } @@ -523,11 +517,11 @@ public override async Task Sign_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE SGN(`b`.`Float`) > 0 +WHERE SIGN(`b`.`Float`) > 0 """, // """ -SELECT SGN(`b`.`Float`) +SELECT SIGN(`b`.`Float`) FROM `BasicTypesEntities` AS `b` """); } @@ -540,10 +534,7 @@ public override async Task Max() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN `b`.`Int` > (`b`.`Short` - CINT(3)) THEN `b`.`Int` - ELSE `b`.`Short` - CINT(3) -END = `b`.`Int` +WHERE GREATEST(`b`.`Int`, `b`.`Short` - CINT(3)) = `b`.`Int` """); } @@ -555,16 +546,7 @@ public override async Task Max_nested() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN CASE - WHEN (`b`.`Short` - CINT(3)) > `b`.`Int` THEN `b`.`Short` - CINT(3) - ELSE `b`.`Int` - END > 1 THEN CASE - WHEN (`b`.`Short` - CINT(3)) > `b`.`Int` THEN `b`.`Short` - CINT(3) - ELSE `b`.`Int` - END - ELSE 1 -END = `b`.`Int` +WHERE GREATEST(`b`.`Short` - CINT(3), `b`.`Int`, 1) = `b`.`Int` """); } @@ -576,28 +558,7 @@ public override async Task Max_nested_twice() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN CASE - WHEN CASE - WHEN 1 > `b`.`Int` THEN 1 - ELSE `b`.`Int` - END > 2 THEN CASE - WHEN 1 > `b`.`Int` THEN 1 - ELSE `b`.`Int` - END - ELSE 2 - END > (`b`.`Short` - CINT(3)) THEN CASE - WHEN CASE - WHEN 1 > `b`.`Int` THEN 1 - ELSE `b`.`Int` - END > 2 THEN CASE - WHEN 1 > `b`.`Int` THEN 1 - ELSE `b`.`Int` - END - ELSE 2 - END - ELSE `b`.`Short` - CINT(3) -END = `b`.`Int` +WHERE GREATEST(1, `b`.`Int`, 2, `b`.`Short` - CINT(3)) = `b`.`Int` """); } @@ -609,10 +570,7 @@ public override async Task Min() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN `b`.`Int` < (`b`.`Short` + CINT(3)) THEN `b`.`Int` - ELSE `b`.`Short` + CINT(3) -END = `b`.`Int` +WHERE LEAST(`b`.`Int`, `b`.`Short` + CINT(3)) = `b`.`Int` """); } @@ -624,16 +582,7 @@ public override async Task Min_nested() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN CASE - WHEN (`b`.`Short` + CINT(3)) < `b`.`Int` THEN `b`.`Short` + CINT(3) - ELSE `b`.`Int` - END < 99999 THEN CASE - WHEN (`b`.`Short` + CINT(3)) < `b`.`Int` THEN `b`.`Short` + CINT(3) - ELSE `b`.`Int` - END - ELSE 99999 -END = `b`.`Int` +WHERE LEAST(`b`.`Short` + CINT(3), `b`.`Int`, 99999) = `b`.`Int` """); } @@ -645,28 +594,7 @@ public override async Task Min_nested_twice() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE CASE - WHEN CASE - WHEN CASE - WHEN 99999 < `b`.`Int` THEN 99999 - ELSE `b`.`Int` - END < 99998 THEN CASE - WHEN 99999 < `b`.`Int` THEN 99999 - ELSE `b`.`Int` - END - ELSE 99998 - END < (`b`.`Short` + CINT(3)) THEN CASE - WHEN CASE - WHEN 99999 < `b`.`Int` THEN 99999 - ELSE `b`.`Int` - END < 99998 THEN CASE - WHEN 99999 < `b`.`Int` THEN 99999 - ELSE `b`.`Int` - END - ELSE 99998 - END - ELSE `b`.`Short` + CINT(3) -END = `b`.`Int` +WHERE LEAST(99999, `b`.`Int`, 99998, `b`.`Short` + CINT(3)) = `b`.`Int` """); } @@ -678,7 +606,7 @@ public override async Task Degrees() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE (`b`.`Double` * (180.0 / 3.141592653589793)) > 0.0 +WHERE DEGREES(`b`.`Double`) > 0.0 """); } @@ -690,7 +618,7 @@ public override async Task Degrees_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE (`b`.`Float` * (180 / 3.1415927)) > 0 +WHERE DEGREES(`b`.`Float`) > 0 """); } @@ -702,7 +630,7 @@ public override async Task Radians() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE (`b`.`Double` * (3.141592653589793 / 180.0)) > 0.0 +WHERE RADIANS(`b`.`Double`) > 0.0 """); } @@ -714,7 +642,7 @@ public override async Task Radians_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE (`b`.`Float` * (3.1415927 / 180)) > 0 +WHERE RADIANS(`b`.`Float`) > 0 """); } @@ -728,7 +656,7 @@ public override async Task Acos() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (1.5707963267948966 + ATN(-`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0))) > 1.0 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND ACOS(`b`.`Double`) > 1.0 """); } @@ -740,7 +668,7 @@ public override async Task Acos_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND (1.5707963267948966 + ATN(-`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1))) > 0.0 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND ACOS(`b`.`Float`) > 0 """); } @@ -755,7 +683,7 @@ public override async Task Asin() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND ATN(`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0)) > -1.7976931348623157E+308 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND ASIN(`b`.`Double`) > -1.7976931348623157E+308 """); } @@ -767,7 +695,7 @@ public override async Task Asin_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND CDBL(ATN(`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1))) > -1.7976931348623157E+308 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND CDBL(ASIN(`b`.`Float`)) > -1.7976931348623157E+308 """); } @@ -782,7 +710,7 @@ public override async Task Atan() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Double`) > 0.0 +WHERE ATAN(`b`.`Double`) > 0.0 """); } @@ -794,7 +722,7 @@ public override async Task Atan_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Float`) > 0 +WHERE ATAN(`b`.`Float`) > 0 """); } @@ -809,7 +737,7 @@ public override async Task Atan2() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Double` / 1.0) > 0.0 +WHERE ATAN2(`b`.`Double`, 1.0) > 0.0 """); } @@ -821,7 +749,7 @@ public override async Task Atan2_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Float` / 1) > 0 +WHERE ATAN2(`b`.`Float`, 1) > 0 """); } @@ -850,7 +778,16 @@ WHERE COS(`b`.`Float`) > 0 } public override async Task Cosh() - => await AssertTranslationFailed(() => base.Cosh()); + { + await base.Cosh(); + + AssertSql( + """ +SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` +FROM `BasicTypesEntities` AS `b` +WHERE COSH(`b`.`Double`) > 0.0 +"""); + } public override async Task Sin() { @@ -877,7 +814,16 @@ WHERE SIN(`b`.`Float`) > 0 } public override async Task Sinh() - => await AssertTranslationFailed(() => base.Sinh()); + { + await base.Sinh(); + + AssertSql( + """ +SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` +FROM `BasicTypesEntities` AS `b` +WHERE SINH(`b`.`Double`) > 0.0 +"""); + } public override async Task Tan() { @@ -904,7 +850,16 @@ WHERE TAN(`b`.`Float`) > 0 } public override async Task Tanh() - => await AssertTranslationFailed(() => base.Tanh()); + { + await base.Tanh(); + + AssertSql( + """ +SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` +FROM `BasicTypesEntities` AS `b` +WHERE TANH(`b`.`Double`) > 0.0 +"""); + } #endregion Trigonometry diff --git a/test/EFCore.LibRed.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs b/test/EFCore.LibRed.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs index f17e5d6b2..4242d5e39 100644 --- a/test/EFCore.LibRed.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs +++ b/test/EFCore.LibRed.FunctionalTests/Query/Translations/MathTranslationsLibRedTest.cs @@ -73,7 +73,7 @@ public override async Task Ceiling() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE IIF(FIX(`b`.`Double`) = `b`.`Double`, FIX(`b`.`Double`), FIX(`b`.`Double`) + 1.0) = 9.0 +WHERE -INT(-`b`.`Double`) = 9.0 """); } @@ -85,7 +85,7 @@ public override async Task Ceiling_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE IIF(FIX(`b`.`Float`) = `b`.`Float`, FIX(`b`.`Float`), FIX(`b`.`Float`) + 1) = 9 +WHERE -INT(-`b`.`Float`) = 9 """); } @@ -97,7 +97,7 @@ public override async Task Floor_decimal() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Decimal`) = 8.0 +WHERE INT(`b`.`Decimal`) = 8.0 """); } @@ -109,7 +109,7 @@ public override async Task Floor_double() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Double`) = 8.0 +WHERE INT(`b`.`Double`) = 8.0 """); } @@ -121,7 +121,7 @@ public override async Task Floor_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE FIX(`b`.`Float`) = 8 +WHERE INT(`b`.`Float`) = 8 """); } @@ -665,7 +665,7 @@ public override async Task Acos() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (1.5707963267948966 + ATN(-`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0))) > 1.0 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (1.5707963267948966 + (ATN(-`b`.`Double` / (SQR(-(`b`.`Double` * `b`.`Double`) + 1.0) + 1.0)) * 2.0)) > 1.0 """); } @@ -677,7 +677,7 @@ public override async Task Acos_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND (1.5707963267948966 + ATN(-`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1))) > 0.0 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND (1.5707963267948966 + (ATN(-`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2.0)) > 0.0 """); } @@ -692,7 +692,7 @@ public override async Task Asin() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND ATN(`b`.`Double` / SQR(-(`b`.`Double` * `b`.`Double`) + 1.0)) > -1.7976931348623157E+308 +WHERE `b`.`Double` >= -1.0 AND `b`.`Double` <= 1.0 AND (ATN(`b`.`Double` / (SQR(-(`b`.`Double` * `b`.`Double`) + 1.0) + 1.0)) * 2.0) > -1.7976931348623157E+308 """); } @@ -704,7 +704,7 @@ public override async Task Asin_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND IIF(ATN(`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1)) IS NULL, NULL, CDBL(ATN(`b`.`Float` / SQR(-(`b`.`Float` * `b`.`Float`) + 1)))) > -1.7976931348623157E+308 +WHERE `b`.`Float` >= -1 AND `b`.`Float` <= 1 AND IIF((ATN(`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2) IS NULL, NULL, CDBL(ATN(`b`.`Float` / (SQR(-(`b`.`Float` * `b`.`Float`) + 1) + 1)) * 2)) > -1.7976931348623157E+308 """); } @@ -746,7 +746,7 @@ public override async Task Atan2() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Double` / 1.0) > 0.0 +WHERE IIF(1.0 = 0.0, SGN(`b`.`Double`) * 1.5707963267948966, ATN(`b`.`Double` / 1.0) + IIF(1.0 < 0.0, IIF(`b`.`Double` < 0.0, -3.141592653589793, 3.141592653589793), 0.0)) > 0.0 """); } @@ -758,7 +758,7 @@ public override async Task Atan2_float() """ SELECT `b`.`Id`, `b`.`Bool`, `b`.`Byte`, `b`.`ByteArray`, `b`.`DateOnly`, `b`.`DateTime`, `b`.`DateTimeOffset`, `b`.`Decimal`, `b`.`Double`, `b`.`Enum`, `b`.`FlagsEnum`, `b`.`Float`, `b`.`Guid`, `b`.`Int`, `b`.`Long`, `b`.`Short`, `b`.`String`, `b`.`TimeOnly`, `b`.`TimeSpan` FROM `BasicTypesEntities` AS `b` -WHERE ATN(`b`.`Float` / 1) > 0 +WHERE IIF(1 = 0, SGN(`b`.`Float`) * 1.5707964, ATN(`b`.`Float` / 1) + IIF(1 < 0, IIF(`b`.`Float` < 0, -3.1415927, 3.1415927), 0)) > 0 """); } diff --git a/test/LibRed.Ado.Tests/LibRedCommandTests.cs b/test/LibRed.Ado.Tests/LibRedCommandTests.cs index 64925b71e..1f2cad20e 100644 --- a/test/LibRed.Ado.Tests/LibRedCommandTests.cs +++ b/test/LibRed.Ado.Tests/LibRedCommandTests.cs @@ -106,6 +106,127 @@ public void DateTime_parameter_is_truncated_to_whole_milliseconds() finally { try { File.Delete(path); } catch (IOException) { } } } + [Fact] + public void A_DateTime2_parameter_keeps_its_ticks() + { + // A DATETIME2 column stores 100-ns ticks, not the OA double, so a parameter that says DbType.DateTime2 is + // passed through whole - as SqlClient passes one - and the same value typed DateTime is still truncated. + string path = Path.Combine(Path.GetTempPath(), $"libred-dt2-{Guid.NewGuid():N}.accdb"); + File.Copy(Northwind, path); + try + { + using var conn = new LibRedConnection($"Data Source={path}"); + conn.Open(); + + using (var create = conn.CreateCommand()) + { create.CommandText = "CREATE TABLE `T` (`Id` INTEGER PRIMARY KEY, `D` DATETIME2)"; create.ExecuteNonQuery(); } + + var subMs = new DateTime(2020, 1, 2, 3, 4, 5, 678).AddTicks(4567); + foreach ((int id, DbType type) in new[] { (1, DbType.DateTime2), (2, DbType.DateTime) }) + { + using var ins = conn.CreateCommand(); + ins.CommandText = "INSERT INTO `T` (`Id`, `D`) VALUES (@id, @d)"; + var i = ins.CreateParameter(); i.ParameterName = "@id"; i.Value = id; ins.Parameters.Add(i); + var p = ins.CreateParameter(); p.ParameterName = "@d"; p.Value = subMs; p.DbType = type; ins.Parameters.Add(p); + Assert.Equal(1, ins.ExecuteNonQuery()); + } + + using (var sel = conn.CreateCommand()) + { + sel.CommandText = "SELECT `D` FROM `T` ORDER BY `Id`"; + using var reader = sel.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(subMs, reader.GetDateTime(0)); + Assert.True(reader.Read()); + Assert.Equal(subMs.AddTicks(-4567), reader.GetDateTime(0)); + } + + // Matched exactly: each parameter finds its own row and only that one. + foreach ((DbType type, int expected) in new[] { (DbType.DateTime2, 1), (DbType.DateTime, 2) }) + { + using var q = conn.CreateCommand(); + q.CommandText = "SELECT SUM(`Id`) FROM `T` WHERE `D` = @d"; + var p = q.CreateParameter(); p.ParameterName = "@d"; p.Value = subMs; p.DbType = type; q.Parameters.Add(p); + Assert.Equal(expected, Convert.ToInt32(q.ExecuteScalar())); + } + } + finally { try { File.Delete(path); } catch (IOException) { } } + } + + [Fact] + public void A_date_plus_a_time_parameter_is_declared_a_date_whatever_the_first_row_holds() + { + // The time becomes a date on the OLE epoch at this boundary, and the engine types the parameter by that + // value — so GetFieldType says DateTime even when the first row's date is Null. + string path = Path.Combine(Path.GetTempPath(), $"libred-ts-{Guid.NewGuid():N}.accdb"); + File.Copy(Northwind, path); + try + { + using var conn = new LibRedConnection($"Data Source={path}"); + conn.Open(); + foreach (string sql in new[] + { + "CREATE TABLE `T` (`Id` INTEGER PRIMARY KEY, `D` DATETIME)", + "INSERT INTO `T` (`Id`, `D`) VALUES (1, NULL)", + "INSERT INTO `T` (`Id`, `D`) VALUES (2, #2020-01-02 12:00:00#)", + }) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + using var q = conn.CreateCommand(); + q.CommandText = "SELECT `D` + @ts FROM `T` ORDER BY `Id`"; + var p = q.CreateParameter(); p.ParameterName = "@ts"; p.Value = TimeSpan.FromHours(6); q.Parameters.Add(p); + using var reader = q.ExecuteReader(); + Assert.Equal(typeof(DateTime), reader.GetFieldType(0)); + Assert.True(reader.Read()); + Assert.True(reader.IsDBNull(0)); + Assert.True(reader.Read()); + Assert.Equal(new DateTime(2020, 1, 2, 18, 0, 0), reader.GetDateTime(0)); + } + finally { try { File.Delete(path); } catch (IOException) { } } + } + + [Fact] + public void A_date_less_a_time_parameter_is_a_date() + { + // A DbType.Time TimeSpan beside a date is a span: less an hour is a date an hour earlier, not the day count a + // date less a date is — in both arms of a UNION ALL, as the report that found it had it. + string path = Path.Combine(Path.GetTempPath(), $"libred-tsminus-{Guid.NewGuid():N}.accdb"); + File.Copy(Northwind, path); + try + { + using var conn = new LibRedConnection($"Data Source={path}"); + conn.Open(); + foreach (string sql in new[] + { + "CREATE TABLE `T` (`Id` INTEGER PRIMARY KEY, `D` DATETIME)", + "INSERT INTO `T` (`Id`, `D`) VALUES (1, #2020-02-29 17:55:00#)", + }) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + using var q = conn.CreateCommand(); + q.CommandText = "SELECT [t].[D] - @ts FROM [T] [t] UNION ALL SELECT [t_1].[D] - @ts FROM [T] [t_1]"; + var p = q.CreateParameter(); + p.ParameterName = "@ts"; p.DbType = DbType.Time; p.Value = TimeSpan.FromHours(1); + q.Parameters.Add(p); + using var reader = q.ExecuteReader(); + Assert.Equal(typeof(DateTime), reader.GetFieldType(0)); + for (int i = 0; i < 2; i++) + { + Assert.True(reader.Read()); + Assert.Equal(new DateTime(2020, 2, 29, 16, 55, 0), reader.GetDateTime(0)); + } + } + finally { try { File.Delete(path); } catch (IOException) { } } + } + [Fact] public void CreateDatabase_creates_a_native_usable_file() { diff --git a/test/LibRed.Core.AccessTests/AccentCollationAccessTests.cs b/test/LibRed.Core.AccessTests/AccentCollationAccessTests.cs index 2389ceafe..c5cb2fef9 100644 --- a/test/LibRed.Core.AccessTests/AccentCollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AccentCollationAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// sorts the accented letter with its base letter's primary weight and records the accent in a secondary /// section. Verified by inserting through LibRed and having Access find the row through the City index. /// +[Collection(AceCollection.Name)] public class AccentCollationAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceAlterColumnTests.cs b/test/LibRed.Core.AccessTests/AceAlterColumnTests.cs index 7855d74dd..ac40ed8b7 100644 --- a/test/LibRed.Core.AccessTests/AceAlterColumnTests.cs +++ b/test/LibRed.Core.AccessTests/AceAlterColumnTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: after LibRed widens a text column (ALTER COLUMN path, via AlterColumn), ACE reads the new max // length and enforces it — a value that fits the new max is accepted, one past it is rejected. +[Collection(AceCollection.Name)] public class AceAlterColumnTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceAlterConstraintTests.cs b/test/LibRed.Core.AccessTests/AceAlterConstraintTests.cs index adbb498c6..db6d8c9b8 100644 --- a/test/LibRed.Core.AccessTests/AceAlterConstraintTests.cs +++ b/test/LibRed.Core.AccessTests/AceAlterConstraintTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: ACE reads and ENFORCES a LibRed-written CHECK (via AddCheckConstraint, the ALTER path) and a // LibRed-written UNIQUE index (via CreateIndex, the ADD CONSTRAINT UNIQUE path). Confirms both write byte-faithful // structures — answering "does AddUnique write it the way ACE does": ACE accepts and enforces it. +[Collection(AceCollection.Name)] public class AceAlterConstraintTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceAutoNumberOverflowRegressionTests.cs b/test/LibRed.Core.AccessTests/AceAutoNumberOverflowRegressionTests.cs index 1d9e91baa..f4857e7e5 100644 --- a/test/LibRed.Core.AccessTests/AceAutoNumberOverflowRegressionTests.cs +++ b/test/LibRed.Core.AccessTests/AceAutoNumberOverflowRegressionTests.cs @@ -20,6 +20,7 @@ namespace LibRed.Core.Tests; // Every case logs what happened (the id assigned, or the engine's own error text) plus the resulting 0x14 // high-water read back through LibRed's catalog, so ACE's and LibRed's behaviour sit side by side in the // output — and then asserts it. Nothing here is asserted that was not first observed against ACE. +[Collection(AceCollection.Name)] public class AceAutoNumberOverflowRegressionTests(ITestOutputHelper output) { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceChooseDefaultTests.cs b/test/LibRed.Core.AccessTests/AceChooseDefaultTests.cs index 1525e7fc4..2935d34fc 100644 --- a/test/LibRed.Core.AccessTests/AceChooseDefaultTests.cs +++ b/test/LibRed.Core.AccessTests/AceChooseDefaultTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: a LibRed-written Choose()/CBool(Choose()) default is read and applied by ACE. Confirms LibRed's // Choose support matches ACE (which has the VBA Choose function), incl. the nested CBool(Choose(...)) form that // ACE's OLE DB DDL parser rejects at CREATE but its expression service applies at insert. +[Collection(AceCollection.Name)] public class AceChooseDefaultTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceCollection.cs b/test/LibRed.Core.AccessTests/AceCollection.cs new file mode 100644 index 000000000..d9eaef56a --- /dev/null +++ b/test/LibRed.Core.AccessTests/AceCollection.cs @@ -0,0 +1,48 @@ +using System.Reflection; +using Xunit; +using Xunit.v3; + +[assembly: LibRed.Core.Tests.ReleaseAceObjectsAfterEachTest] + +namespace LibRed.Core.Tests; + +/// +/// Groups the classes that drive the real ACE OLE DB provider, so xunit runs them one after another. ACE +/// faults under concurrent use — two such classes running in parallel throw +/// SEHException: External component has thrown an exception at the same millisecond and take the test +/// process down with 0xC0000005. The same collection as LibRed.Engine.AccessTests uses. +/// +/// Every class in this project reaches ACE, so every class carries the attribute. Add it to any new +/// one. Serialising the classes is not the whole of it: see . +[CollectionDefinition(Name)] +public sealed class AceCollection +{ + public const string Name = "ACE OLE DB"; +} + +/// +/// The opt-in for the ACE surveys: tools that measure ACE over a large space — every DAO collating order, say — +/// and report rather than assert. They drive DAO heavily and prove nothing in an ordinary build, so they run only +/// when asked for with LIBRED_ACE_SURVEYS=1. +/// +internal static class AceSurveys +{ + public const string Variable = "LIBRED_ACE_SURVEYS"; + + public static void RequireOptIn() => + Assert.SkipUnless(Environment.GetEnvironmentVariable(Variable) == "1", + $"An ACE survey, not a test: set {Variable}=1 to run it."); +} + +/// +/// Releases, as each test ends, the COM objects it left to the finalizer — the DAO objects the probes never +/// release, whose locals a Debug build keeps alive to the end of the test — so their teardown inside ACE happens +/// now, with no test running, rather than at the next GC in the middle of a later one. See +/// for what that collision does. +/// +[AttributeUsage(AttributeTargets.Assembly)] +public sealed class ReleaseAceObjectsAfterEachTestAttribute : BeforeAfterTestAttribute +{ + public override void After(MethodInfo methodUnderTest, IXunitTest test) => + AceTestDatabase.ReleaseAbandonedComObjects(); +} diff --git a/test/LibRed.Core.AccessTests/AceCompoundDefaultTests.cs b/test/LibRed.Core.AccessTests/AceCompoundDefaultTests.cs index a5d75d96e..36a4c6ae5 100644 --- a/test/LibRed.Core.AccessTests/AceCompoundDefaultTests.cs +++ b/test/LibRed.Core.AccessTests/AceCompoundDefaultTests.cs @@ -11,6 +11,7 @@ namespace LibRed.Core.Tests; // a front-end limitation only: ACE's expression *service*, used when reading a stored default, evaluates the // full expression. So a compound default that LibRed writes straight to LvProp (bypassing ACE's DDL parser) is // read and applied by ACE on insert. Verifies LibRed's SQL surface is a superset of ACE's DDL here. +[Collection(AceCollection.Name)] public class AceCompoundDefaultTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceCreatedDatabaseTests.cs b/test/LibRed.Core.AccessTests/AceCreatedDatabaseTests.cs index f4c02a126..b2543baf2 100644 --- a/test/LibRed.Core.AccessTests/AceCreatedDatabaseTests.cs +++ b/test/LibRed.Core.AccessTests/AceCreatedDatabaseTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// A database LibRed creates from scratch (no DAO/ADOX) is opened, queried, and written by real Access. +[Collection(AceCollection.Name)] public class AceCreatedDatabaseTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceDateTime2UpgradeTests.cs b/test/LibRed.Core.AccessTests/AceDateTime2UpgradeTests.cs index 8087c468b..b36eeb1df 100644 --- a/test/LibRed.Core.AccessTests/AceDateTime2UpgradeTests.cs +++ b/test/LibRed.Core.AccessTests/AceDateTime2UpgradeTests.cs @@ -10,11 +10,12 @@ namespace LibRed.Core.Tests; // DAO-created ACE 12 baseline; the only other byte either arm touched was the opening user's commit slot, which // moves for any write at all. See docs/format/page-00-database.md. // -// Requires DAO and the ACE OLE DB provider; skips when DAO is absent, as the other ACE probes do. ACE +// Starts from a Northwind copy, which is already the ACE 12 format (version byte 0x02); no DAO. ACE // heap-corrupts (0xC0000374) under connection churn in this shape, reproducibly, and takes the test process // with it - so each phase uses ONE connection for all of its statements. A connection is only reopened where // the file has to be closed in between. -public class AceDateTime2UpgradeTests(ITestOutputHelper output) +[Collection(AceCollection.Name)] +public class AceDateTime2UpgradeTests { // The guard for the half of the finding LibRed would come to depend on: that the byte is SUFFICIENT, not // merely necessary. If a future ACE wanted a companion flag, LibRed would be silently writing files Access @@ -23,7 +24,7 @@ public class AceDateTime2UpgradeTests(ITestOutputHelper output) [Fact] public void Writing_the_version_byte_is_a_complete_upgrade_to_datetime2() { - if (!TryCreateAce12Database("dt2-upgrade-", out string path)) return; + string path = CopyAce12Database("dt2-upgrade-"); try { Assert.Equal(0x02, VersionByte(path)); @@ -94,7 +95,7 @@ public void LibRed_decodes_datetime2_values_that_ace_reads_back_wrongly() ("#2020-02-29 00:00:00#", new DateTime(2020, 2, 29)), ]; - if (!TryCreateAce12Database("dt2-decode-", out string path)) return; + string path = CopyAce12Database("dt2-decode-"); try { SetVersionByte(path, 0x06); @@ -119,9 +120,9 @@ public void LibRed_decodes_datetime2_values_that_ace_reads_back_wrongly() finally { TemporaryDatabase.Delete(path); } } - /// Creates an ACE 12 (version byte 0x02) database through DAO — the format LibRed itself - /// creates. Returns false, having reported it, when DAO is not installed. - private bool TryCreateAce12Database(string prefix, out string path) + /// A Northwind copy: an ACE 12 (version byte 0x02) database, the format LibRed itself creates, + /// that already holds data. + private static string CopyAce12Database(string prefix) { // Both tests here have ACE itself create or read a DATETIME2 column, which an ACE below 17 cannot do // at all — CI installs the 2016 redistributable. Skip rather than fail: a machine without the type @@ -130,24 +131,7 @@ private bool TryCreateAce12Database(string prefix, out string path) AceTestDatabase.SupportsColumnType(TestDatabases.NorthwindAccdb, "DATETIME2"), AceTestDatabase.UnsupportedColumnTypeReason("DATETIME2")); - path = ""; - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable - skipped."); return false; } - - path = TemporaryDatabase.CreatePath(prefix); - File.Delete(path); // DAO creates the file itself and refuses an existing one - - // 128 == dbVersion120, the ACE 12 / Access 2007 format. - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - return true; + return TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, prefix); } private static void Execute(DbConnection connection, string sql) @@ -178,7 +162,4 @@ private static void SetVersionByte(string path, byte version) stream.Seek(0x14, SeekOrigin.Begin); stream.WriteByte(version); } - - private static object? Invoke(object target, string member, params object?[] args) => - target.GetType().InvokeMember(member, System.Reflection.BindingFlags.InvokeMethod, null, target, args); } diff --git a/test/LibRed.Core.AccessTests/AceDdlOnLibRedDatabaseProbeTest.cs b/test/LibRed.Core.AccessTests/AceDdlOnLibRedDatabaseProbeTest.cs index c0db6796c..aef956e66 100644 --- a/test/LibRed.Core.AccessTests/AceDdlOnLibRedDatabaseProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AceDdlOnLibRedDatabaseProbeTest.cs @@ -12,10 +12,12 @@ namespace LibRed.Core.Tests; // ANSWERED: the file was missing MSysComplexColumns. ACE consults it whenever it creates a catalog object — // CREATE TABLE and CREATE VIEW, and only those; DML, CREATE INDEX, ALTER ADD COLUMN and DROP TABLE all work // without it. DatabaseCreator.CreateEmpty now writes it (and the nine MSysComplexType_* tables) for version -// >= 0x02. See docs/format/system-catalog.md. +// >= 0x02. See docs/format/system-catalog.md. The DAO probes that isolated it — dropping one system table at a +// time from a DAO-created database — are in git history. // -// Ace_runs_ddl_against_a_libred_created_database is the regression guard and asserts; the rest report. -// Keep the guard: LibRed reading its own file back proves nothing about whether Access will accept it. +// Ace_runs_ddl_against_a_libred_created_database is the regression guard and asserts. Keep it: LibRed reading +// its own file back proves nothing about whether Access will accept it. +[Collection(AceCollection.Name)] public class AceDdlOnLibRedDatabaseProbeTest(ITestOutputHelper output) { [Theory] @@ -75,222 +77,6 @@ public void Probe_system_tables_libred_creates_versus_access() finally { TemporaryDatabase.Delete(libred); } } - // The control: DAO creates a database through the real engine, and its files are known to omit the - // NavPane/AccessStorage tables that Access adds on first open. If ACE will run DDL in a DAO-created - // database, those tables are not the blocker and the difference is elsewhere. - [Fact] - public void Probe_ace_ddl_against_a_dao_created_database() - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath("ace-ddl-dao-"); - File.Delete(path); // DAO creates the file itself and refuses an existing one - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - output.WriteLine($"DAO-created system tables: {string.Join(", ", SystemTables(path))}"); - - using var connection = AceTestDatabase.Open(path); - try - { - using var command = connection.CreateCommand(); - command.CommandText = "CREATE TABLE AceMade (K TEXT(30), V LONG)"; - command.ExecuteNonQuery(); - output.WriteLine(" CREATE TABLE OK -> the missing system tables are NOT the blocker"); - } - catch (Exception ex) { output.WriteLine($" CREATE TABLE {ex.GetType().Name}: {ex.Message.Trim()}"); } - } - finally { TemporaryDatabase.Delete(path); } - } - - // Isolate the required table: start from a DAO-created database (where ACE DDL works), drop one system - // table with LibRed, and see whether ACE then refuses. Whatever flips it is what CREATE TABLE needs. - [Theory] - [InlineData("MSysComplexColumns")] - [InlineData("MSysComplexType_Text")] - [InlineData("MSysQueries")] - public void Probe_which_system_table_ace_ddl_needs(string drop) - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath($"ace-ddl-drop-"); - File.Delete(path); - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - try - { - using var db = JetDatabase.Open(path, readOnly: false); - db.DropTable(drop); - } - catch (Exception ex) { output.WriteLine($"dropping {drop}: {ex.GetType().Name}: {ex.Message.Trim()}"); return; } - - using var connection = AceTestDatabase.Open(path); - try - { - using var command = connection.CreateCommand(); - command.CommandText = "CREATE TABLE AceMade (K TEXT(30), V LONG)"; - command.ExecuteNonQuery(); - output.WriteLine($"without {drop,-22} CREATE TABLE still OK"); - } - catch (Exception ex) { output.WriteLine($"without {drop,-22} CREATE TABLE {ex.GetType().Name}: {ex.Message.Trim()}"); } - } - finally { TemporaryDatabase.Delete(path); } - } - - // Does ACE actually WRITE to MSysComplexColumns when it creates an ordinary table, or does it only need - // the table to exist? That decides whether LibRed must populate it or merely provide an empty one — and - // the dumped schema is what LibRed would have to build. - [Fact] - public void Probe_what_ace_writes_to_complex_columns() - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath("complex-cols-"); - File.Delete(path); - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - DumpSchema(path); - output.WriteLine($"rows before any DDL: {RowCount(path)}"); - - using (var connection = AceTestDatabase.Open(path)) - foreach (string sql in new[] - { - "CREATE TABLE Plain (K TEXT(30), V LONG)", - "CREATE TABLE Typed (A LONG, B TEXT(20), C MEMO, D DATETIME, E CURRENCY, F GUID, G OLEOBJECT)", - "CREATE INDEX IX_Plain ON Plain (K)", - }) - { - using var command = connection.CreateCommand(); - command.CommandText = sql; - command.ExecuteNonQuery(); - output.WriteLine($"after {sql[..Math.Min(34, sql.Length)],-36} rows = {RowCount(path)}"); - } - } - finally { TemporaryDatabase.Delete(path); } - } - - // NOT probed by dropping indexes: removing MSysComplexColumns' indexes with LibRed and reopening makes - // ACE fault natively (0xC0000005) rather than report anything, so that route says the file is malformed, - // not whether the indexes are required. The constructive test — build the table *without* indexes in - // DatabaseCreator and see whether ACE DDL works — belongs with the implementation. - - // WHAT is ACE doing with MSysComplexColumns? It reads it and never writes it, so the question is which - // operations need it at all. If only CREATE fails, ACE consults it per-creation; if the whole DDL surface - // fails while DML keeps working, it is binding the table as part of a fixed system-table set and the - // "lookup" is really a bind of the catalog the DDL path expects to exist. - [Fact] - public void Probe_which_operations_need_complex_columns() - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath("complex-need-"); - File.Delete(path); - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - // Pre-build the objects the later statements act on, while the registry still exists. - using (var setup = AceTestDatabase.Open(path)) - foreach (string sql in new[] - { - "CREATE TABLE Existing (K TEXT(30), V LONG)", - "CREATE TABLE Doomed (K TEXT(30))", - "CREATE TABLE Altered (K TEXT(30))", - "INSERT INTO Existing (K, V) VALUES ('a', 1)", - }) - { - using var command = setup.CreateCommand(); - command.CommandText = sql; - command.ExecuteNonQuery(); - } - - using (var db = JetDatabase.Open(path, readOnly: false)) db.DropTable("MSysComplexColumns"); - output.WriteLine("dropped MSysComplexColumns; now exercising the surface:"); - - using var connection = AceTestDatabase.Open(path); - foreach ((string what, string sql) in new[] - { - ("SELECT", "SELECT COUNT(*) FROM Existing"), - ("INSERT", "INSERT INTO Existing (K, V) VALUES ('b', 2)"), - ("UPDATE", "UPDATE Existing SET V = 3 WHERE K = 'a'"), - ("DELETE", "DELETE FROM Existing WHERE K = 'b'"), - ("CREATE TABLE", "CREATE TABLE Fresh (K TEXT(30))"), - ("CREATE INDEX", "CREATE INDEX IX_Existing ON Existing (K)"), - ("ALTER ADD COL", "ALTER TABLE Altered ADD COLUMN Extra LONG"), - ("CREATE VIEW", "CREATE VIEW V1 AS SELECT K FROM Existing"), - ("DROP TABLE", "DROP TABLE Doomed"), - }) - { - using var command = connection.CreateCommand(); - command.CommandText = sql; - Exception? error = Record.Exception(() => command.ExecuteNonQuery()); - output.WriteLine($" {what,-14} {(error is null ? "OK" : error.Message.Trim())}"); - } - } - finally { TemporaryDatabase.Delete(path); } - } - - private static int RowCount(string path) - { - using var db = JetDatabase.Open(path); - return db.OpenTable("MSysComplexColumns").Rows().Count(); - } - - private void DumpSchema(string path) - { - using var db = JetDatabase.Open(path); - TableDef table = db.Catalog.Tables.Single(t => t.Name == "MSysComplexColumns"); - output.WriteLine("MSysComplexColumns schema:"); - foreach (ColumnDef c in table.Columns.OrderBy(c => c.Index)) - output.WriteLine($" {c.Index} {c.Name,-16} {c.Type,-12} len={c.Length,-4} fixed={c.IsFixedLength} nullable={c.IsNullable}"); - foreach (IndexDef i in table.Indexes) - output.WriteLine($" index {i.Name,-16} unique={i.IsUnique} pk={i.IsPrimaryKey} cols={string.Join(",", i.Columns.Select(c => c.Column.Name))}"); - } - - private static object? Invoke(object target, string member, params object?[] args) => - target.GetType().InvokeMember(member, System.Reflection.BindingFlags.InvokeMethod, null, target, args); - private static string[] SystemTables(string path) { using var db = JetDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceDecimalDeclarationProbeTest.cs b/test/LibRed.Core.AccessTests/AceDecimalDeclarationProbeTest.cs new file mode 100644 index 000000000..a81fa0a3b --- /dev/null +++ b/test/LibRed.Core.AccessTests/AceDecimalDeclarationProbeTest.cs @@ -0,0 +1,157 @@ +using System.Data.OleDb; +using LibRed; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +// What precision and scale does ACE actually stamp on a NUMERIC/DECIMAL column's descriptor? +// +// TdefBuilder.EffectivePrecision resolves a ColumnSpec's default Precision of 0 to 18 before writing, on the +// grounds that 18 is ACE's own default for a bare DECIMAL. That figure was inherited from AccessTypeMapper +// (`column.Size ?? 18`) rather than measured, so this measures it: ACE creates the columns, LibRed reads the +// descriptor bytes back. +[Collection(AceCollection.Name)] +public class AceDecimalDeclarationProbeTest(ITestOutputHelper output) +{ + [Fact] + public void What_ace_stamps_for_each_decimal_declaration() + { + (string Column, string Declared)[] cases = + [ + ("Bare", "DECIMAL"), + ("BareNumeric", "NUMERIC"), + ("PrecOnly", "DECIMAL(10)"), + ("PrecScale", "DECIMAL(10,3)"), + ("MaxPrec", "DECIMAL(28,4)"), + ]; + + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-decl-"); + try + { + using (OleDbConnection connection = AceTestDatabase.Open(path)) + { + using OleDbCommand create = connection.CreateCommand(); + create.CommandText = + $"CREATE TABLE DecDecl ({string.Join(", ", cases.Select(c => $"{c.Column} {c.Declared}"))})"; + create.ExecuteNonQuery(); + } + + using var db = JetDatabase.Open(path, readOnly: true); + TableDef table = db.Catalog.Tables.Single(t => t.Name == "DecDecl"); + foreach ((string column, string declared) in cases) + { + ColumnDef def = table.Columns.Single(c => c.Name == column); + output.WriteLine( + $"{declared,-15} -> type {def.Type}, precision {def.Precision}, scale {def.Scale}, " + + $"length {def.Length}, fixed {def.IsFixedLength}"); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + /// + /// Whether ACE will accept an out-of-range precision or scale at all — asked directly, by declaring one. + /// The 1..28 / 0..p bounds LibRed enforces were taken from AccessTypeMapper, and the earlier probe + /// only tried declarations that were already in range, so it could not tell "ACE refuses 0" from "nobody + /// asked for 0". + /// + [Theory] + [InlineData("DECIMAL(0)")] + [InlineData("DECIMAL(0,0)")] + [InlineData("DECIMAL(29)")] // one past the maximum precision + [InlineData("DECIMAL(5,7)")] // scale beyond its own precision + [InlineData("DECIMAL(28,28)")] // scale equal to precision — the legal edge + [InlineData("DECIMAL(1,0)")] // the minimum + public void Whether_ace_accepts_an_out_of_range_decimal_declaration(string declared) + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-range-"); + try + { + using (OleDbConnection connection = AceTestDatabase.Open(path)) + { + using OleDbCommand create = connection.CreateCommand(); + create.CommandText = $"CREATE TABLE DecRange (V {declared})"; + try { create.ExecuteNonQuery(); } + catch (Exception ex) + { + output.WriteLine($"{declared,-15} -> ACE REFUSES: {ex.Message.Trim()}"); + return; + } + } + + using var db = JetDatabase.Open(path, readOnly: true); + ColumnDef def = db.Catalog.Tables.Single(t => t.Name == "DecRange").Columns.Single(c => c.Name == "V"); + output.WriteLine( + $"{declared,-15} -> ACE ACCEPTS, stamps precision {def.Precision}, scale {def.Scale}"); + } + finally { TemporaryDatabase.Delete(path); } + } + + /// + /// What a DECIMAL(28,28) can actually hold. ACE accepts the declaration, but all 28 digits sit after + /// the point, so there are none in front and every value must be under 1 — a column it is easier to declare + /// than to use. Worth confirming ACE agrees, because LibRed's guard derives the limit arithmetically + /// (10^(p-s), here 10^0 = 1) and a declaration nothing can store would make that untestable. + /// + [Theory] + [InlineData("0.5")] + [InlineData("0.1234567890123456789012345678")] // all 28 decimals used + [InlineData("0.9999999999999999999999999999")] // the largest value that fits + [InlineData("1")] // the first that does not + public void What_a_decimal_28_28_can_hold(string literal) + { + var value = decimal.Parse(literal, System.Globalization.CultureInfo.InvariantCulture); + + string acePath = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-2828-"); + try + { + using OleDbConnection connection = AceTestDatabase.Open(acePath); + using (OleDbCommand create = connection.CreateCommand()) + { + create.CommandText = "CREATE TABLE Dec2828 (Id LONG, V DECIMAL(28,28))"; + create.ExecuteNonQuery(); + } + + try + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = $"INSERT INTO Dec2828 (Id, V) VALUES (1, {literal})"; + insert.ExecuteNonQuery(); + + using OleDbCommand back = connection.CreateCommand(); + back.CommandText = "SELECT CStr(V) FROM Dec2828 WHERE Id = 1"; + output.WriteLine($"{literal,-32} ACE stores {back.ExecuteScalar()}"); + } + catch (Exception ex) { output.WriteLine($"{literal,-32} ACE REFUSES: {ex.Message.Trim()}"); } + } + finally { TemporaryDatabase.Delete(acePath); } + + string path = TemporaryDatabase.CreatePath("libred-2828-"); + File.Delete(path); + try + { + Storage.DatabaseCreator.CreateEmpty(path); + using var db = JetDatabase.Open(path, readOnly: false); + db.CreateTable("D", + [ + new ColumnSpec("V", JetDataType.FixedPoint, 17, IsFixedLength: true, Precision: 28, Scale: 28), + ]); + + try + { + db.OpenTable("D").Insert([value]); + output.WriteLine($"{literal,-32} LibRed stores {db.OpenTable("D").Rows().First()[0]}"); + } + catch (Exception ex) + { + output.WriteLine($"{literal,-32} LibRed REFUSES: {ex.GetType().Name}: {ex.Message.Trim()}"); + } + } + finally { TemporaryDatabase.Delete(path); } + } + +} diff --git a/test/LibRed.Core.AccessTests/AceDefaultExpressionLimitsTests.cs b/test/LibRed.Core.AccessTests/AceDefaultExpressionLimitsTests.cs index 44ee9e761..e3445095e 100644 --- a/test/LibRed.Core.AccessTests/AceDefaultExpressionLimitsTests.cs +++ b/test/LibRed.Core.AccessTests/AceDefaultExpressionLimitsTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // aggregates (DCount) too — all rejected as "Unknown function". // - The 255-char cap is a DAO-API limit, NOT an engine/file-format limit: ACE accepts and applies a 300+ char // default expression. LibRed writes such defaults to LvProp and they round-trip. +[Collection(AceCollection.Name)] public class AceDefaultExpressionLimitsTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceNullArgumentProbeTest.cs b/test/LibRed.Core.AccessTests/AceNullArgumentProbeTest.cs index 4e6831faa..11e4aaa91 100644 --- a/test/LibRed.Core.AccessTests/AceNullArgumentProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AceNullArgumentProbeTest.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // guard came along for free. EF 11 elides the redundant conversion, so MID now receives a NULL length directly // and ACE errors (the GearsOfWar Null_semantics_..._optional_navigation_complex failures). Guarding has to move // to the functions themselves, so this establishes which arguments actually need it. +[Collection(AceCollection.Name)] public class AceNullArgumentRegressionTests(ITestOutputHelper output) { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AcePreEpochDateProbeTest.cs b/test/LibRed.Core.AccessTests/AcePreEpochDateProbeTest.cs index f387b2872..08a36bc2e 100644 --- a/test/LibRed.Core.AccessTests/AcePreEpochDateProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AcePreEpochDateProbeTest.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // ACE may well have inherited the same weirdness, in which case LibRed is already bug-compatible and should stay // that way. This probe establishes which it is. The existing DateAdd/DateDiff functional tests do not cover it: // they all use modern (Northwind-era) dates, where the serial is positive and the anomaly cannot appear. +[Collection(AceCollection.Name)] public class AcePreEpochDateRegressionTests(ITestOutputHelper output) { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceRenameTests.cs b/test/LibRed.Core.AccessTests/AceRenameTests.cs index 2573c7679..16b0f8688 100644 --- a/test/LibRed.Core.AccessTests/AceRenameTests.cs +++ b/test/LibRed.Core.AccessTests/AceRenameTests.cs @@ -11,6 +11,7 @@ namespace LibRed.Core.Tests; // own malformed bytes back consistently) but fatal to Access. So the checks below are deliberately made // *through ACE*: it resolves the new names, honours the carried-over DEFAULT, and still enforces the // relationship whose by-name references were repointed. +[Collection(AceCollection.Name)] public class AceRenameTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceSingleDoubleCompareProbeTest.cs b/test/LibRed.Core.AccessTests/AceSingleDoubleCompareProbeTest.cs index b37a4027e..c10afaddb 100644 --- a/test/LibRed.Core.AccessTests/AceSingleDoubleCompareProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AceSingleDoubleCompareProbeTest.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // "compare in single precision when a float is involved" rule matches ACE for the column-vs-column case, or // whether a column-type-aware fix is needed. Reports counts; the assertions just pin what we observed so a // future ACE change is noticed. +[Collection(AceCollection.Name)] public class AceSingleDoubleCompareRegressionTests(ITestOutputHelper output) { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceSmuggledColRefDefaultTests.cs b/test/LibRed.Core.AccessTests/AceSmuggledColRefDefaultTests.cs index 7d7255859..40af5ea39 100644 --- a/test/LibRed.Core.AccessTests/AceSmuggledColRefDefaultTests.cs +++ b/test/LibRed.Core.AccessTests/AceSmuggledColRefDefaultTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; // the column-reference prohibition is enforced by ACE's expression service at INSERT (evaluation) time, not by // its DDL parser. So ACE inserts nothing — it rejects the row with the same "field in a default" error it gives // at create time. There is no way to smuggle a working column-ref default past the engine. +[Collection(AceCollection.Name)] public class AceSmuggledColRefDefaultTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceSwitchDefaultTests.cs b/test/LibRed.Core.AccessTests/AceSwitchDefaultTests.cs index c88a08c5c..5522c0dd4 100644 --- a/test/LibRed.Core.AccessTests/AceSwitchDefaultTests.cs +++ b/test/LibRed.Core.AccessTests/AceSwitchDefaultTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: a LibRed-written Switch() default is read and applied by ACE (which has the VBA Switch // function), confirming LibRed's Switch matches ACE. +[Collection(AceCollection.Name)] public class AceSwitchDefaultTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AceVbaConversionProbeTest.cs b/test/LibRed.Core.AccessTests/AceVbaConversionProbeTest.cs index 57e23f6c7..bee63bc02 100644 --- a/test/LibRed.Core.AccessTests/AceVbaConversionProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AceVbaConversionProbeTest.cs @@ -24,6 +24,7 @@ namespace LibRed.Core.Tests; // // Output is written to the test log; the assertions pin only what has actually been observed, so a future ACE // change (or a wrong assumption on our side) is noticed rather than silently absorbed. +[Collection(AceCollection.Name)] public class AceVbaConversionRegressionTests(ITestOutputHelper output) { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ActionQueryProcedureAccessTests.cs b/test/LibRed.Core.AccessTests/ActionQueryProcedureAccessTests.cs index f22fb7722..91aabb2bb 100644 --- a/test/LibRed.Core.AccessTests/ActionQueryProcedureAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ActionQueryProcedureAccessTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; /// row; an append (INSERT … VALUES) query stores the target table (Attribute=1/Flag 3) plus one /// Attribute=6/Flag 0x8000 row per column. Access recognises and runs both. /// +[Collection(AceCollection.Name)] public class ActionQueryProcedureAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AddColumnAccessTests.cs b/test/LibRed.Core.AccessTests/AddColumnAccessTests.cs index 6c2267f48..f264cbd34 100644 --- a/test/LibRed.Core.AccessTests/AddColumnAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AddColumnAccessTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // ADD COLUMN is a metadata TDEF edit (inverse of DROP COLUMN): the descriptor/name are appended, the new id // comes from the 0x29 high-water, counts bump, rows are untouched (read NULL). Verify ACE opens a // LibRed-column-added file, reads existing rows with the new column NULL, and can insert using it. +[Collection(AceCollection.Name)] public class AddColumnAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AddIndexToPopulatedTableAccessTests.cs b/test/LibRed.Core.AccessTests/AddIndexToPopulatedTableAccessTests.cs index f4cd77db1..cd305831f 100644 --- a/test/LibRed.Core.AccessTests/AddIndexToPopulatedTableAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AddIndexToPopulatedTableAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// key). LibRed back-fills the new index B-tree from the existing rows and appends its usage-map row /// without disturbing the data/other-index maps; Access reads every row and enforces the key. /// +[Collection(AceCollection.Name)] public class AddIndexToPopulatedTableAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AlterForeignKeyAccessTests.cs b/test/LibRed.Core.AccessTests/AlterForeignKeyAccessTests.cs index 4f6061de4..cf89e498d 100644 --- a/test/LibRed.Core.AccessTests/AlterForeignKeyAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AlterForeignKeyAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// child backing index, the parent's incoming relationship block and the MSysRelationships rows; Access /// reads the relationship and enforces referential integrity. /// +[Collection(AceCollection.Name)] public class AlterForeignKeyAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/AlterPrimaryKeyAccessTests.cs b/test/LibRed.Core.AccessTests/AlterPrimaryKeyAccessTests.cs index 2189be184..c7b6bb543 100644 --- a/test/LibRed.Core.AccessTests/AlterPrimaryKeyAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AlterPrimaryKeyAccessTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; /// A multi-column primary key added to an (empty) table — the write path ALTER TABLE ADD CONSTRAINT /// PRIMARY KEY reuses (CreateIndex with isPrimary/isUnique). Access accepts it and enforces uniqueness. /// +[Collection(AceCollection.Name)] public class AlterPrimaryKeyAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ArabicShaddaCollationAccessTests.cs b/test/LibRed.Core.AccessTests/ArabicShaddaCollationAccessTests.cs index e6d521e9b..04a20b845 100644 --- a/test/LibRed.Core.AccessTests/ArabicShaddaCollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ArabicShaddaCollationAccessTests.cs @@ -16,6 +16,7 @@ namespace LibRed.Core.Tests; // // So the samples here are all CONTEXT: the same mark after a letter, after another mark, after itself, and // with nothing at all in front of it. A per-character test cannot fail on this bug, which is the whole point. +[Collection(AceCollection.Name)] public class ArabicShaddaCollationAccessTests { private const char Shadda = (char)0x0651; diff --git a/test/LibRed.Core.AccessTests/AstralCollationProbeTest.cs b/test/LibRed.Core.AccessTests/AstralCollationProbeTest.cs index 06c0b5c64..32b791a01 100644 --- a/test/LibRed.Core.AccessTests/AstralCollationProbeTest.cs +++ b/test/LibRed.Core.AccessTests/AstralCollationProbeTest.cs @@ -14,6 +14,7 @@ namespace LibRed.Core.Tests; // what this measures. // // Both probes need ACE and are opt-in via LIBRED_ASTRAL=1. +[Collection(AceCollection.Name)] public class AstralCollationProbeTest(ITestOutputHelper output) { /// diff --git a/test/LibRed.Core.AccessTests/AuditRegressionAccessTests.cs b/test/LibRed.Core.AccessTests/AuditRegressionAccessTests.cs index f83773614..fd8628301 100644 --- a/test/LibRed.Core.AccessTests/AuditRegressionAccessTests.cs +++ b/test/LibRed.Core.AccessTests/AuditRegressionAccessTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // Regressions for the spec-vs-code audit. Each one is a defect that shipped, so each is pinned by the // smallest sequence that reproduced it rather than by a unit test of the fix — most of these were bugs // precisely because a single path looked correct in isolation and only diverged from its sibling. +[Collection(AceCollection.Name)] public class AuditRegressionAccessTests(ITestOutputHelper output) { // ------------------------------------------------------------------ format version @@ -46,32 +47,58 @@ public void Raising_a_jet_mdb_to_an_ACE_version_is_refused() finally { TemporaryDatabase.Delete(path); } } - // The counterpart: raising an ACCDB is fine, and ACE still opens the result. A 2010-format file carries - // 0x15 = 0x01, and the raise moves only 0x14 — a (0x05, 0x01) pair the spec had never observed, so this - // measures that ACE accepts it rather than assuming so. - [Fact] - public void Raising_a_2010_format_accdb_leaves_the_minor_byte_and_ACE_still_opens_it() + // The counterpart: raising an ACCDB. A 2010-format file carries 0x15 = 0x01, and the raise used to move + // only 0x14, leaving a (0x05, 0x01) pair ACE opens but never writes: ACE's own raise clears the minor. One + // base file, copied, so ACE's raise and LibRed's start from identical bytes and page 0 can be compared whole + // — everything but the commit-byte table (§2.2), which moves for any write. + [Theory] + [InlineData("BIGINT", JetVersion.Version16_2016, 0x05)] + [InlineData("DATETIME2", JetVersion.Version17_2019, 0x06)] + public void Raising_a_2010_format_accdb_rewrites_page_zero_as_ACE_does( + string typeName, JetVersion target, byte expectedVersion) { - string path = TemporaryDatabase.CreatePath("raise-accdb-"); + Assert.SkipUnless(AceTestDatabase.SupportsColumnType(TestDatabases.NorthwindAccdb, typeName), + AceTestDatabase.UnsupportedColumnTypeReason(typeName)); + + string basePath = TemporaryDatabase.CreatePath("raise-base-"); + string acePath = TemporaryDatabase.CreatePath("raise-ace-"); + string libPath = TemporaryDatabase.CreatePath("raise-lib-"); try { - DatabaseCreator.CreateEmpty(path, version: 0x03); - Assert.Equal(0x01, PageZero(path, 0x15)); - - using (var db = JetDatabase.Open(path, readOnly: false)) - Assert.True(db.EnsureFormatAtLeast(JetVersion.Version16_2016)); - - Assert.Equal(0x05, PageZero(path, 0x14)); - Assert.Equal(0x01, PageZero(path, 0x15)); // untouched by the raise - - using var connection = AceTestDatabase.Open(path); - Exec(connection, "CREATE TABLE AfterRaise (K LONG, V TEXT(20))"); - Exec(connection, "INSERT INTO AfterRaise (K, V) VALUES (1, 'ok')"); - using var read = connection.CreateCommand(); - read.CommandText = "SELECT V FROM AfterRaise WHERE K = 1"; - Assert.Equal("ok", read.ExecuteScalar()); + DatabaseCreator.CreateEmpty(basePath, version: 0x03); + Assert.Equal(0x01, PageZero(basePath, JetFormatBase.MinorVersionOffset)); + File.Copy(basePath, acePath, overwrite: true); + File.Copy(basePath, libPath, overwrite: true); + + using (var connection = AceTestDatabase.Open(acePath)) + Exec(connection, $"CREATE TABLE Raised (K {typeName})"); + + using (var db = JetDatabase.Open(libPath, readOnly: false)) + Assert.True(db.EnsureFormatAtLeast(target)); + + byte[] ace = PageZeroBytes(acePath), lib = PageZeroBytes(libPath); + Assert.Equal(expectedVersion, ace[JetFormatBase.VersionOffset]); + Assert.Equal(0x00, ace[JetFormatBase.MinorVersionOffset]); + var differences = Enumerable.Range(0, CommitByteTableStart) + .Where(i => ace[i] != lib[i]) + .Select(i => $"0x{i:X3} ace={ace[i]:X2} lib={lib[i]:X2}") + .ToList(); + Assert.True(differences.Count == 0, string.Join("; ", differences)); + + // And ACE works in the file LibRed raised. + using var reopened = AceTestDatabase.Open(libPath); + Exec(reopened, $"CREATE TABLE AfterRaise (K LONG, V {typeName})"); + Exec(reopened, "INSERT INTO AfterRaise (K) VALUES (1)"); + using var read = reopened.CreateCommand(); + read.CommandText = "SELECT COUNT(*) FROM AfterRaise"; + Assert.Equal(1, Convert.ToInt32(read.ExecuteScalar())); + } + finally + { + TemporaryDatabase.Delete(basePath); + TemporaryDatabase.Delete(acePath); + TemporaryDatabase.Delete(libPath); } - finally { TemporaryDatabase.Delete(path); } } // ------------------------------------------------------------------ foreign keys @@ -353,10 +380,15 @@ private static void Insert(JetDatabase db, string table, params (string Column, t.Insert(values); } - private static byte PageZero(string path, int offset) + private static byte PageZero(string path, int offset) => PageZeroBytes(path)[offset]; + + /// Page 0 starts its user commit-byte table here; every write moves a slot in it. + private const int CommitByteTableStart = 0xE00; + + private static byte[] PageZeroBytes(string path) { using var channel = PageChannel.Open(path, readOnly: true); - return channel.ReadPage(0).Span[offset]; + return channel.ReadPage(0).Span.ToArray(); } private static void Exec(OleDbConnection connection, string sql) diff --git a/test/LibRed.Core.AccessTests/AutoNumberSeedTests.cs b/test/LibRed.Core.AccessTests/AutoNumberSeedTests.cs index 147a15503..fc45ded00 100644 --- a/test/LibRed.Core.AccessTests/AutoNumberSeedTests.cs +++ b/test/LibRed.Core.AccessTests/AutoNumberSeedTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // 0x14 high-water to the *last inserted value* (not the max), so the next auto id re-derives from it and // collides with an existing row — the "duplicate values in the index/primary key" error. LibRed diverges (it // advances 0x14 monotonically and is immune — see AutoNumberSeedImmunityTests in LibRed.Engine.Tests). +[Collection(AceCollection.Name)] public class AutoNumberSeedTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/BigIntKeyEncodingTests.cs b/test/LibRed.Core.AccessTests/BigIntKeyEncodingTests.cs index 0cead87bd..955e9dada 100644 --- a/test/LibRed.Core.AccessTests/BigIntKeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/BigIntKeyEncodingTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // // Northwind is ACE 12 (version byte 0x02); Large Number needs ACE 16 (0x05). Unlike the DATETIME2 tests // nothing flips the byte by hand here — ACE is left to do it, because whether it does was the open question. +[Collection(AceCollection.Name)] public class BigIntKeyEncodingTests { // Both extremes and both signs. The key transform is a sign-bit flip, so a sample of positives would pass diff --git a/test/LibRed.Core.AccessTests/BitwiseOperatorAccessTests.cs b/test/LibRed.Core.AccessTests/BitwiseOperatorAccessTests.cs index 97a54c432..d7e01b1f1 100644 --- a/test/LibRed.Core.AccessTests/BitwiseOperatorAccessTests.cs +++ b/test/LibRed.Core.AccessTests/BitwiseOperatorAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// Access bitwise operators (BAND / BOR / BXOR infix, BNOT prefix) — the same operator syntax runs in both /// LibRed's engine and ACE and yields the same results. /// +[Collection(AceCollection.Name)] public class BitwiseOperatorAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/BooleanWriteAccessTests.cs b/test/LibRed.Core.AccessTests/BooleanWriteAccessTests.cs index 06d282acc..a533101ca 100644 --- a/test/LibRed.Core.AccessTests/BooleanWriteAccessTests.cs +++ b/test/LibRed.Core.AccessTests/BooleanWriteAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// (set = true); LibRed coerces the inserted value (1/-1/0/TRUE/FALSE) with Access truthiness. Access reads /// the bits back correctly and a bare-boolean predicate returns the right rows. /// +[Collection(AceCollection.Name)] public class BooleanWriteAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/CalculatedColumnAccessTests.cs b/test/LibRed.Core.AccessTests/CalculatedColumnAccessTests.cs index ae9b88500..75cadc359 100644 --- a/test/LibRed.Core.AccessTests/CalculatedColumnAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CalculatedColumnAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // LibRed cannot create a calculated column (Access SQL has no syntax for one), so DAO's object model is // the author here — the same path Access's UI uses. Each column gets its own table because ACE validates // the expression when the TableDef is appended, and one rejected expression would take the rest with it. +[Collection(AceCollection.Name)] public class CalculatedColumnAccessTests(ITestOutputHelper output) { private const int UseJet = 2; @@ -518,6 +519,60 @@ public void Creates_a_calculated_column_ace_accepts( finally { TemporaryDatabase.Delete(path); } } + // The raise a calculated column forces lands ON the 2010 format, whose created files carry minor byte 0x01 + // -- yet ACE's raise writes 0x00 there. One 2007 base, copied, so ACE's raise and LibRed's start from + // identical bytes and page 0 can be compared whole -- everything but the commit-byte table from 0xE00, + // which moves for any write. + [Fact] + public void A_calculated_column_raises_page_zero_as_ACE_does() + { + object? engine = CreateDbEngine(); + Assert.SkipWhen(engine is null, "DAO is unavailable in this process; it authors the fixture."); + + string basePath = TemporaryDatabase.CreatePath("calc-raise-base-"); + string acePath = TemporaryDatabase.CreatePath("calc-raise-ace-"); + string libPath = TemporaryDatabase.CreatePath("calc-raise-lib-"); + try + { + DatabaseCreator.CreateEmpty(basePath); + File.Copy(basePath, acePath, overwrite: true); + File.Copy(basePath, libPath, overwrite: true); + + object workspace = Invoke(engine!, "CreateWorkspace", "", "admin", "", UseJet)!; + object database = Invoke(workspace, "OpenDatabase", acePath)!; + AppendCalculatedTable(database, "CLong", DbLong, 0, "[Qty]*2"); + Invoke(database, "Close"); + + using (var db = JetDatabase.Open(libPath, readOnly: false)) + db.CreateTable("T_CLong", [ + new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("Qty", JetDataType.Int32, 4, IsFixedLength: true), + ColumnSpec.Calculated("CLong", JetDataType.Int32, "[Qty]*2"), + ]); + + byte[] ace = PageZero(acePath), lib = PageZero(libPath); + output.WriteLine($"ACE: version 0x{ace[0x14]:X2} minor 0x{ace[0x15]:X2}"); + output.WriteLine($"lib: version 0x{lib[0x14]:X2} minor 0x{lib[0x15]:X2}"); + var differences = Enumerable.Range(0, 0xE00) + .Where(i => ace[i] != lib[i]) + .Select(i => $"0x{i:X3} ace={ace[i]:X2} lib={lib[i]:X2}") + .ToList(); + Assert.True(differences.Count == 0, string.Join("; ", differences)); + } + finally + { + TemporaryDatabase.Delete(basePath); + TemporaryDatabase.Delete(acePath); + TemporaryDatabase.Delete(libPath); + } + } + + private static byte[] PageZero(string path) + { + using var channel = LibRed.IO.PageChannel.Open(path, readOnly: true); + return channel.ReadPage(0).Span.ToArray(); + } + // Validation is mandatory, not a courtesy: an expression ACE rejects produces a column it refuses to read // at all, so LibRed must never author one. These are the four refusal shapes ACE has. [Theory] @@ -879,17 +934,7 @@ private static bool Matches(object? ace, object? libred) _ => $"{value} ({value.GetType().Name})", }; - private static object? CreateDbEngine() - { - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { return Activator.CreateInstance(type); } - catch (Exception) { /* registered but not instantiable in this bitness */ } - } - return null; - } + private static object? CreateDbEngine() => AceTestDatabase.CreateDaoEngine(); // What ACE writes for a conversion over Null. CDbl does NOT propagate Null the way every other function // here does -- the VBA conversions raise on it -- so ACE is caching an error state rather than a value, @@ -1522,6 +1567,39 @@ void Apply(string label, Action operation) finally { TemporaryDatabase.Delete(path); } } + // A Memo/OLE retype takes the full RewriteColumn path rather than the in-place descriptor edit. + // That rebuild must carry the calculated column's LvProp entries as well as its descriptor: the + // descriptor's calculated flag alone is not sufficient for ACE to know the cached payload's type. + [Fact] + public void Access_reads_a_calculated_column_after_an_unrelated_full_column_rewrite() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "calc-rewrite-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("CalcRewrite", + [ + new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("Qty", JetDataType.Int32, 4, IsFixedLength: true), + ColumnSpec.Calculated("TwiceQty", JetDataType.Int32, "[Qty] * 2"), + new ColumnSpec("Notes", JetDataType.Memo, 0, IsFixedLength: false), + ], primaryKey: ["Id"]); + db.OpenTable("CalcRewrite").Insert([1, 7, null, "before rewrite"]); + + // Memo -> Text cannot be an in-place edit, so it exercises RewriteColumn. + db.AlterColumn("CalcRewrite", "Notes", + new ColumnSpec("Notes", JetDataType.Text, 200, IsFixedLength: false)); + } + + using var connection = AceTestDatabase.Open(path); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT TwiceQty FROM CalcRewrite WHERE Id = 1"; + Assert.Equal(14, Convert.ToInt32(command.ExecuteScalar())); + } + finally { TemporaryDatabase.Delete(path); } + } + /// Opens the database with DAO, applies to one field of a saved /// TableDef, and reports what happened. private static string DaoField(object workspace, string path, string table, string column, Action mutate) diff --git a/test/LibRed.Core.AccessTests/ChainedLongValueAccessTests.cs b/test/LibRed.Core.AccessTests/ChainedLongValueAccessTests.cs index 593376651..ab2439f15 100644 --- a/test/LibRed.Core.AccessTests/ChainedLongValueAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ChainedLongValueAccessTests.cs @@ -1,6 +1,8 @@ using System.Data.OleDb; using LibRed; using LibRed.Catalog; +using LibRed.Pages; +using LibRed.Storage; using Xunit; namespace LibRed.Core.Tests; @@ -10,6 +12,7 @@ namespace LibRed.Core.Tests; /// each chunk row beginning with a 4-byte pointer to the next. This checks a large value round-trips /// through LibRed and that Access reads it back intact. /// +[Collection(AceCollection.Name)] public class ChainedLongValueAccessTests { // 20 000 chars = 40 000 bytes → several chained LVAL pages (chunk data is 4072 bytes/page). @@ -49,4 +52,97 @@ [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), } finally { TemporaryDatabase.Delete(path); } } + + // A chained descriptor's 0x08 and the FIRST chain page's header 0x08 are one value in two places, and + // ACE enforces that they agree: patch either alone and it refuses the record as "you and another user + // are attempting to change the same data at the same time". The value itself is free -- ACE stamps + // GetTickCount(), LibRed writes zero, and both are accepted because both are self-consistent. What a + // writer must never do is set one without the other, which is what this pins: half a stamp is a file + // Access cannot read, and nothing else in our tests would notice. + [Fact] + public void Libreds_chain_stamp_agrees_with_its_first_chain_page() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "chain-stamp-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("Big", + [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("M", JetDataType.Memo, 0, IsFixedLength: false)], + primaryKey: ["Id"]); + db.OpenTable("Big").Insert([1, Big]); + } + + using var channel = LibRed.IO.PageChannel.Open(path); + TableDef definition = new JetCatalog(channel).FindTable("Big")!; + var decoder = new RowDecoder(definition.Columns, channel.Format); + int columnId = definition.Columns.First(c => c.Name == "M").ColumnId; + + byte[] descriptor = new UsageMap(channel, definition).DataPages() + .Select(p => { var page = new DataPage(); page.Read(channel.ReadPage(p), channel.Format); return page; }) + .SelectMany(page => Enumerable.Range(0, page.RowCount) + .Where(row => !page.Rows[row].IsDeleted) + .SelectMany(row => decoder.LongValueRaw(page.GetRow(row)))) + .Single(d => d.Key == columnId).Value[..12]; + + Assert.Equal(0x00, descriptor[3] & 0xC0); // chained, or the stamp would not apply + int firstChainPage = descriptor[5] | (descriptor[6] << 8) | (descriptor[7] << 16); + + var header = new byte[channel.PageSize]; + channel.ReadPage(firstChainPage, header); + Assert.Equal(Convert.ToHexString(descriptor[8..12]), Convert.ToHexString(header[8..12])); + // And it is a real tag, not two zeroes agreeing by accident — zero would satisfy the check while + // proving nothing, which is what LibRed used to write. + Assert.NotEqual("00000000", Convert.ToHexString(descriptor[8..12])); + } + finally { TemporaryDatabase.Delete(path); } + } + + // The check has to bite, or writing the stamp is decoration. Breaks the agreement the way a stale + // pointer would — the chain rewritten under a descriptor that still names its first page — and expects + // the read to refuse rather than hand back another value's bytes. + [Fact] + public void A_chain_whose_stamp_disagrees_is_refused() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "chain-stale-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("Big", + [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("M", JetDataType.Memo, 0, IsFixedLength: false)], + primaryKey: ["Id"]); + db.OpenTable("Big").Insert([1, Big]); + } + + int firstChainPage; + using (var channel = LibRed.IO.PageChannel.Open(path, readOnly: false)) + { + TableDef definition = new JetCatalog(channel).FindTable("Big")!; + var decoder = new RowDecoder(definition.Columns, channel.Format); + int columnId = definition.Columns.First(c => c.Name == "M").ColumnId; + int dataPage = new UsageMap(channel, definition).DataPages().First(); + var parsed = new DataPage(); + parsed.Read(channel.ReadPage(dataPage), channel.Format); + byte[] descriptor = decoder.LongValueRaw(parsed.GetRow(0)).Single(d => d.Key == columnId).Value; + firstChainPage = descriptor[5] | (descriptor[6] << 8) | (descriptor[7] << 16); + + // Restamp the chain page alone, as a rewrite by another writer would. + var page = new byte[channel.PageSize]; + channel.ReadPage(firstChainPage, page); + BitConverter.GetBytes(0xDEADBEEFu).CopyTo(page, channel.Format.DataChainStampOffset); + channel.WritePage(firstChainPage, page); + } + + using var database = JetDatabase.Open(path); + var table = database.OpenTable("Big"); + int m = table.Definition.Columns.First(c => c.Name == "M").Index; + var thrown = Assert.Throws(() => table.Rows().Select(r => r[m]).ToList()); + Assert.Contains("not the one this row was written against", thrown.Message); + Assert.Contains("DEADBEEF", thrown.Message); + } + finally { TemporaryDatabase.Delete(path); } + } } diff --git a/test/LibRed.Core.AccessTests/CollationSurveyProbeTests.cs b/test/LibRed.Core.AccessTests/CollationSurveyProbeTests.cs index 674b3c840..48b1fd282 100644 --- a/test/LibRed.Core.AccessTests/CollationSurveyProbeTests.cs +++ b/test/LibRed.Core.AccessTests/CollationSurveyProbeTests.cs @@ -37,10 +37,15 @@ namespace LibRed.Core.Tests; // Results are written to %TEMP%\libred-collation-survey (override with LIBRED_SURVEY_OUT) as well as to the // test output, because the departure lists are long. // -// Run: dotnet test test\LibRed.Core.Tests\LibRed.Core.Tests.csproj +// A survey rather than a test, and the heaviest DAO user in the suite, so both passes run only when asked for +// (AceSurveys): +// +// Run: set LIBRED_ACE_SURVEYS=1 +// dotnet test test\LibRed.Core.AccessTests\LibRed.Core.AccessTests.csproj // --filter "FullyQualifiedName~CollationSurveyProbeTests.Survey_dao_acceptance" -// dotnet test test\LibRed.Core.Tests\LibRed.Core.Tests.csproj +// dotnet test test\LibRed.Core.AccessTests\LibRed.Core.AccessTests.csproj // --filter "FullyQualifiedName~CollationSurveyProbeTests.Survey_keys_batch_04" +[Collection(AceCollection.Name)] public class CollationSurveyProbeTests(ITestOutputHelper output) { private const int UseJet = 2; @@ -247,6 +252,7 @@ private sealed record Candidate(int LangId, string Name, string Script, int Batc [Fact] public void Survey_dao_acceptance() { + AceSurveys.RequireOptIn(); object? engine = CreateDbEngine(out string progId); Assert.SkipWhen(engine is null, "DAO is not available in this process."); object workspace = Invoke(engine!, "CreateWorkspace", "", "admin", "", UseJet)!; @@ -586,13 +592,15 @@ public void Every_batch_has_a_test_method() private void SurveyBatch(int batch) { + AceSurveys.RequireOptIn(); + // Whether there is anything to measure is settled before DAO is touched, so an empty batch costs nothing. + Candidate[] members = [.. KeyCandidates().Where(c => c.Batch == batch)]; + Assert.SkipWhen(members.Length == 0, $"batch {batch} is empty"); + object? engine = CreateDbEngine(out string progId); Assert.SkipWhen(engine is null, "DAO is not available in this process."); object workspace = Invoke(engine!, "CreateWorkspace", "", "admin", "", UseJet)!; - Candidate[] members = [.. KeyCandidates().Where(c => c.Batch == batch)]; - Assert.SkipWhen(members.Length == 0, $"batch {batch} is empty"); - string[] samples = Samples(members.Select(m => m.Script)); var report = new StringBuilder(); Write(report, $"batch {batch:00} - DAO engine {progId}, {samples.Length} samples, " + @@ -1049,6 +1057,7 @@ private static void Exec(System.Data.OleDb.OleDbConnection connection, string sq private static object? CreateDbEngine(out string progId) { + AceTestDatabase.ReleaseAbandonedComObjects(); foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) { progId = $"DAO.DBEngine.{n}"; diff --git a/test/LibRed.Core.AccessTests/CollationV1PatchProbeTests.cs b/test/LibRed.Core.AccessTests/CollationV1PatchProbeTests.cs index 3667d2d27..1ed722b82 100644 --- a/test/LibRed.Core.AccessTests/CollationV1PatchProbeTests.cs +++ b/test/LibRed.Core.AccessTests/CollationV1PatchProbeTests.cs @@ -32,6 +32,7 @@ namespace LibRed.Core.Tests; // // This found Romanian v1, which nothing else could have. What is left is the instrument and the control: // the surveys it drove are done, and their results are recorded in page-03-04 §10.4 and page-02b §3.4. +[Collection(AceCollection.Name)] public class CollationV1PatchProbeTests(ITestOutputHelper output) { private static readonly Collation GeneralV1 = Collation.General; diff --git a/test/LibRed.Core.AccessTests/CollationVersionDiffProbeTest.cs b/test/LibRed.Core.AccessTests/CollationVersionDiffProbeTest.cs index bef892e68..f661762be 100644 --- a/test/LibRed.Core.AccessTests/CollationVersionDiffProbeTest.cs +++ b/test/LibRed.Core.AccessTests/CollationVersionDiffProbeTest.cs @@ -14,6 +14,7 @@ namespace LibRed.Core.Tests; // // The v1 database comes from LIBRED_V1_PROBE — Access itself must create it (Access.Application // .NewCurrentDatabase honours the "New Database Sort Order" option; DAO ignores it and always writes v0). +[Collection(AceCollection.Name)] public class CollationVersionDiffProbeTest(ITestOutputHelper output) { private static readonly string[] Samples = diff --git a/test/LibRed.Core.AccessTests/ColumnAliasViewAccessTests.cs b/test/LibRed.Core.AccessTests/ColumnAliasViewAccessTests.cs index 0812051af..10b84c9ea 100644 --- a/test/LibRed.Core.AccessTests/ColumnAliasViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ColumnAliasViewAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// A view column can carry an alias, stored in the MSysQueries column row's Name1 (as Access does, e.g. /// Invoices' Customers.CompanyName AS CustomerName). Access resolves the aliased output column. /// +[Collection(AceCollection.Name)] public class ColumnAliasViewAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ColumnIdHighWaterTests.cs b/test/LibRed.Core.AccessTests/ColumnIdHighWaterTests.cs index 9635150aa..e6a9cea2f 100644 --- a/test/LibRed.Core.AccessTests/ColumnIdHighWaterTests.cs +++ b/test/LibRed.Core.AccessTests/ColumnIdHighWaterTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // The column-id high-water (TDEF 0x29) never decrements on DROP COLUMN, so once 255 ids have been handed out // no further column can be added — even when the *live* count is lower — until the database is compacted. // ACE enforces this ("Too many fields defined"); LibRed must too, rather than write a 256th id ACE can't read. +[Collection(AceCollection.Name)] public class ColumnIdHighWaterTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ColumnWidthLimitAccessTests.cs b/test/LibRed.Core.AccessTests/ColumnWidthLimitAccessTests.cs index d68c2a52c..983e45546 100644 --- a/test/LibRed.Core.AccessTests/ColumnWidthLimitAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ColumnWidthLimitAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // fields (65535). ACE's real limit is the 4060-byte record cap applied to the widest row the declaration // permits, and a table over it makes the whole database unopenable - "Unrecognized database format", the // same damage as a 100-character rename. A plain CreateTable of 252 GUID columns did exactly that. +[Collection(AceCollection.Name)] public class ColumnWidthLimitAccessTests : TempDatabaseTest { private static ColumnSpec Id => new("Id", JetDataType.Int32, 4, IsFixedLength: true); diff --git a/test/LibRed.Core.AccessTests/CommitByteTableTests.cs b/test/LibRed.Core.AccessTests/CommitByteTableTests.cs index 68f55b05e..a0f876bb7 100644 --- a/test/LibRed.Core.AccessTests/CommitByteTableTests.cs +++ b/test/LibRed.Core.AccessTests/CommitByteTableTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // // ONE connection for the whole sequence — ACE heap-corrupts under connection churn (0xC0000374) and takes the // test process with it. +[Collection(AceCollection.Name)] public class CommitByteTableTests { private const int Slot1 = 0xE02; // 0xE00 is slot 0 (exclusive mode); slot 1 is the first shared user @@ -120,16 +121,7 @@ public void Compacting_restarts_the_counter_but_reopening_carries_it() } } - private static object? DaoEngine() - { - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { return Activator.CreateInstance(type); } catch (Exception) { } - } - return null; - } + private static object? DaoEngine() => AceTestDatabase.CreateDaoEngine(); private static object? Invoke(object target, string member, params object?[] args) => target.GetType().InvokeMember(member, System.Reflection.BindingFlags.InvokeMethod, null, target, args); diff --git a/test/LibRed.Core.AccessTests/ComplexSystemTableLayoutProbeTest.cs b/test/LibRed.Core.AccessTests/ComplexSystemTableLayoutProbeTest.cs deleted file mode 100644 index 553e04da0..000000000 --- a/test/LibRed.Core.AccessTests/ComplexSystemTableLayoutProbeTest.cs +++ /dev/null @@ -1,77 +0,0 @@ -using LibRed; -using LibRed.Catalog; -using Xunit; - -namespace LibRed.Core.Tests; - -// PROBE: the exact shape of the complex-type system tables in a database the real engine created, so -// DatabaseCreator can reproduce them rather than approximate them — page numbers, column layout, indexes, -// and the MSysObjects catalog rows. -// -// Complex columns (multi-value / attachment) arrived with Access 2007 / ACE 12, so these tables exist only -// from version byte 0x02 up; a Jet 4 (.mdb) database has none of them. -public class ComplexSystemTableLayoutProbeTest(ITestOutputHelper output) -{ - [Fact] - public void Probe_complex_system_table_layout() - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath("complex-layout-"); - File.Delete(path); - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - using var db = JetDatabase.Open(path); - output.WriteLine($"file: {new FileInfo(path).Length / 4096} pages of 4096"); - output.WriteLine(""); - - output.WriteLine("page table rows indexes"); - foreach (TableDef t in db.Catalog.Tables.OrderBy(t => t.DefinitionPage)) - output.WriteLine($"{t.DefinitionPage,4} {t.Name,-30} {db.OpenTable(t.Name).Rows().Count(),4} " + - $"{string.Join(", ", t.Indexes.Select(i => $"{i.Name}({string.Join("+", i.Columns.Select(c => c.Column.Name))}{(i.IsUnique ? ",unique" : "")}{(i.IsPrimaryKey ? ",pk" : "")})"))}"); - - output.WriteLine(""); - foreach (TableDef t in db.Catalog.Tables.Where(t => t.Name.StartsWith("MSysComplex", StringComparison.OrdinalIgnoreCase)) - .OrderBy(t => t.DefinitionPage)) - { - output.WriteLine($"== {t.Name} (page {t.DefinitionPage})"); - foreach (ColumnDef c in t.Columns.OrderBy(c => c.Index)) - output.WriteLine($" {c.Index} {c.Name,-22} {c.Type,-12} len={c.Length,-4} fixed={c.IsFixedLength,-5} " + - $"nullable={c.IsNullable,-5} id={c.ColumnId} auto={c.IsAutoNumber}"); - foreach (IndexDef i in t.Indexes) - output.WriteLine($" index {i.Name,-24} root={i.RootPage} unique={i.IsUnique} pk={i.IsPrimaryKey} " + - $"required={i.Required} ignoreNulls={i.IgnoreNulls} cols={string.Join(",", i.Columns.Select(c => $"{c.Column.Name}{(c.Ascending ? "" : " DESC")}"))}"); - } - - output.WriteLine(""); - output.WriteLine("MSysObjects rows for the complex tables (Id, ParentId, Name, Type, Flags):"); - var objects = db.OpenTable("MSysObjects"); - var def = objects.Definition; - int idIdx = def.FindColumn("Id")!.Index, parentIdx = def.FindColumn("ParentId")!.Index; - int nameIdx = def.FindColumn("Name")!.Index, typeIdx = def.FindColumn("Type")!.Index; - int flagsIdx = def.FindColumn("Flags")!.Index; - foreach (object?[] row in objects.Rows()) - { - string name = (string?)row[nameIdx] ?? ""; - if (!name.StartsWith("MSys", StringComparison.OrdinalIgnoreCase)) continue; - output.WriteLine($" Id={Convert.ToInt32(row[idIdx]),6} ParentId=0x{Convert.ToInt32(row[parentIdx]):X8} " + - $"{name,-30} Type={row[typeIdx]} Flags=0x{Convert.ToInt32(row[flagsIdx] ?? 0):X8}"); - } - } - finally { TemporaryDatabase.Delete(path); } - } - - private static object? Invoke(object target, string member, params object?[] args) => - target.GetType().InvokeMember(member, System.Reflection.BindingFlags.InvokeMethod, null, target, args); -} diff --git a/test/LibRed.Core.AccessTests/CompositeFkNullAccessTests.cs b/test/LibRed.Core.AccessTests/CompositeFkNullAccessTests.cs index 2c4bc560b..5e7c57065 100644 --- a/test/LibRed.Core.AccessTests/CompositeFkNullAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CompositeFkNullAccessTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; // Ground truth for LibRed's composite-FK enforcement: ACE applies MATCH FULL — a partial null in a composite // foreign key (one column null, the other not) is rejected, only all-null or a fully-matching key is allowed. // (SQL Server's MATCH SIMPLE would skip the check when any column is null; ACE does not.) +[Collection(AceCollection.Name)] public class CompositeFkNullAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/CompositeIndexOrderingAccessTests.cs b/test/LibRed.Core.AccessTests/CompositeIndexOrderingAccessTests.cs index 513164568..4711dc031 100644 --- a/test/LibRed.Core.AccessTests/CompositeIndexOrderingAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CompositeIndexOrderingAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// Checks mixed-type, mixed-direction composite index keys against a live ACE-created B-tree. +[Collection(AceCollection.Name)] public class CompositeIndexOrderingAccessTests { [Fact] diff --git a/test/LibRed.Core.AccessTests/CompressedTextAccessTests.cs b/test/LibRed.Core.AccessTests/CompressedTextAccessTests.cs index 8b4bb7ec4..aa419b013 100644 --- a/test/LibRed.Core.AccessTests/CompressedTextAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CompressedTextAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // to a chained value; and it applies only when the column is declared WITH COMPRESSION, every character // fits one byte, and it actually saves space (the 2-byte FF FE marker means 1- and 2-character values stay // UTF-16, and compression starts at 3). +[Collection(AceCollection.Name)] public class CompressedTextAccessTests : TempDatabaseTest { [Theory] diff --git a/test/LibRed.Core.AccessTests/ContractionProbeTest.cs b/test/LibRed.Core.AccessTests/ContractionProbeTest.cs index 42c948da9..a1c4bb7b6 100644 --- a/test/LibRed.Core.AccessTests/ContractionProbeTest.cs +++ b/test/LibRed.Core.AccessTests/ContractionProbeTest.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // So: for each order, the component letters on their own, every digraph, the doubled forms (Hungarian writes // a doubled digraph by doubling only its first letter - "ggy" is "gy"+"gy", not "g"+"gy"), and real words. // Printed in full, no capping, so the structure is visible rather than inferred. +[Collection(AceCollection.Name)] public class ContractionProbeTest(ITestOutputHelper output) { private static readonly (string Fixture, string[] Samples)[] Cases = diff --git a/test/LibRed.Core.AccessTests/CounterSeedIncrementAccessTests.cs b/test/LibRed.Core.AccessTests/CounterSeedIncrementAccessTests.cs index 81d31607f..b6467f855 100644 --- a/test/LibRed.Core.AccessTests/CounterSeedIncrementAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CounterSeedIncrementAccessTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // Byte-faithful check: a LibRed-written COUNTER(seed, increment) is read by Access — it opens the file without // repair, and continues the AutoNumber sequence from the seed with the custom increment. +[Collection(AceCollection.Name)] public class CounterSeedIncrementAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/CreateTableAccessTests.cs b/test/LibRed.Core.AccessTests/CreateTableAccessTests.cs index 602ae6e02..6d4b537fb 100644 --- a/test/LibRed.Core.AccessTests/CreateTableAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CreateTableAccessTests.cs @@ -5,6 +5,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class CreateTableAccessTests { private static string CopyToTemp() diff --git a/test/LibRed.Core.AccessTests/CreateViewAccessTests.cs b/test/LibRed.Core.AccessTests/CreateViewAccessTests.cs index f4a3a6380..4b8c7ed3a 100644 --- a/test/LibRed.Core.AccessTests/CreateViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CreateViewAccessTests.cs @@ -5,6 +5,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class CreateViewAccessTests { private static string CopyToTemp() diff --git a/test/LibRed.Core.AccessTests/CreatedDatabaseCollationAccessTests.cs b/test/LibRed.Core.AccessTests/CreatedDatabaseCollationAccessTests.cs index 3de5584f7..e3a4b8bda 100644 --- a/test/LibRed.Core.AccessTests/CreatedDatabaseCollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CreatedDatabaseCollationAccessTests.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // The bar is not that the file opens. It is that ACE will CREATE AN INDEX in it and write keys that match // LibRed's own — two engines agreeing on a shared index, which is the only check that catches a wrong key, // since a disagreement does not error, it just makes seeks miss rows. +[Collection(AceCollection.Name)] public class CreatedDatabaseCollationAccessTests(ITestOutputHelper output) { /// Every collation LibRed claims to encode, found by asking rather than by keeping a list that diff --git a/test/LibRed.Core.AccessTests/CreatedJet4DatabaseAccessTests.cs b/test/LibRed.Core.AccessTests/CreatedJet4DatabaseAccessTests.cs index 99454ec80..cf0bd0f92 100644 --- a/test/LibRed.Core.AccessTests/CreatedJet4DatabaseAccessTests.cs +++ b/test/LibRed.Core.AccessTests/CreatedJet4DatabaseAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // LibRed creates the Access 2000 / 2002-2003 `.mdb` (Jet 4, version byte 0x01) as well as the ACCDB formats. // Creating it is only half the claim — ACE has to open the result, read what LibRed wrote, and write into it // itself, which is what separates a plausible file from a valid one. +[Collection(AceCollection.Name)] public class CreatedJet4DatabaseAccessTests(ITestOutputHelper output) { [Fact] diff --git a/test/LibRed.Core.AccessTests/DaoDatabaseCreationProbeTest.cs b/test/LibRed.Core.AccessTests/DaoDatabaseCreationProbeTest.cs deleted file mode 100644 index 44a66981a..000000000 --- a/test/LibRed.Core.AccessTests/DaoDatabaseCreationProbeTest.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System.Reflection; -using LibRed.Catalog; -using LibRed; -using Xunit; - -namespace LibRed.Core.Tests; - -// PROBE: can DAO's DBEngine.CreateDatabase produce a database with the Access-2010 "General" (v1) sort order? -// -// The v1 weight table is the one piece of the text collation LibRed cannot encode, and the blocker is that no -// v1 database exists to reverse-engineer. Access's UI can make one ("New database sort order" = General), but -// that is a manual step. DAO is the programmatic creator the repo already uses (DaoDatabaseCreator), so the -// question is whether its connect string or database-type argument can select the sort version too — the -// documented string carries only LANGID/CP/COUNTRY, i.e. the locale, not the version. -// -// Each attempt is created, then opened with LibRed to read the sort-order version byte (page 0, 0x71). -public class DaoDatabaseCreationProbeTest(ITestOutputHelper output) -{ - private const int UseJet = 2; - - [Fact] - public void Probe_dao_created_database_sort_order_versions() - { - object? engine = CreateDbEngine(out string progId); - if (engine is null) - { - output.WriteLine("No DAO.DBEngine ProgID could be instantiated — DAO is unavailable in this process."); - return; - } - - output.WriteLine($"DAO engine: {progId}"); - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", UseJet)!; - - // (label, connect string, database-type argument). 0 = omit the argument entirely. - (string Label, string Connect, int Type)[] attempts = - [ - ("default (ACE 12)", ";LANGID=0x0409;CP=1252;COUNTRY=0", 128), - ("no type argument", ";LANGID=0x0409;CP=1252;COUNTRY=0", 0), - ("Jet 4", ";LANGID=0x0409;CP=1252;COUNTRY=0", 64), - ("general-legacy spelled out", ";LANGID=0x0409;CP=1252;COUNTRY=0;SORTORDER=GeneralLegacy", 128), - ("general spelled out", ";LANGID=0x0409;CP=1252;COUNTRY=0;SORTORDER=General", 128), - ("NLS version requested", ";LANGID=0x0409;CP=1252;COUNTRY=0;NLSVERSION=1", 128), - ]; - - foreach ((string label, string connect, int type) in attempts) - { - string path = TemporaryDatabase.CreatePath("dao-probe-"); - try - { - object database = type > 0 - ? Invoke(workspace, "CreateDatabase", path, connect, type)! - : Invoke(workspace, "CreateDatabase", path, connect)!; - Invoke(database, "Close"); - - using var db = JetDatabase.Open(path); - output.WriteLine($" {label,-28} -> created; collation version = {db.DefaultCollationVersion}"); - } - catch (TargetInvocationException ex) - { - output.WriteLine($" {label,-28} -> rejected: {ex.InnerException?.Message.Trim()}"); - } - catch (Exception ex) - { - output.WriteLine($" {label,-28} -> {ex.GetType().Name}: {ex.Message.Trim()}"); - } - finally { TemporaryDatabase.Delete(path); } - } - } - - // CompactDatabase is the documented way to give an existing database a different collating order: it takes - // a destination locale. If the sort *version* can be selected anywhere in DAO, this is the other candidate. - [Fact] - public void Probe_dao_compact_with_destination_locale() - { - object? engine = CreateDbEngine(out string progId); - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - output.WriteLine($"DAO engine: {progId}"); - - string[] locales = - [ - ";LANGID=0x0409;CP=1252;COUNTRY=0", - ";LANGID=0x0409;CP=1252;COUNTRY=0;SORTORDER=General", - ";LANGID=0x0809;CP=1252;COUNTRY=0", // en-GB, a different LANGID entirely - ]; - - foreach (string locale in locales) - { - string source = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "dao-compact-src-"); - string destination = TemporaryDatabase.CreatePath("dao-compact-dst-"); - try - { - Invoke(engine, "CompactDatabase", source, destination, locale); - using var db = JetDatabase.Open(destination); - output.WriteLine($" {locale,-52} -> version {db.DefaultCollationVersion}, lcid {db.DefaultCollationLcid}"); - } - catch (TargetInvocationException ex) - { - output.WriteLine($" {locale,-52} -> rejected: {ex.InnerException?.Message.Trim()}"); - } - catch (Exception ex) - { - output.WriteLine($" {locale,-52} -> {ex.GetType().Name}: {ex.Message.Trim()}"); - } - finally { TemporaryDatabase.Delete(source); TemporaryDatabase.Delete(destination); } - } - } - - // Read the sort order of a database created by Access itself (via Access.Application.NewCurrentDatabase), - // which — unlike DAO — honours the application's "New Database Sort Order" option. Path comes from the - // LIBRED_V1_PROBE environment variable so the COM automation stays outside the test. - [Fact] - public void Probe_sort_order_of_an_access_created_database() - { - string? path = Environment.GetEnvironmentVariable("LIBRED_V1_PROBE"); - if (path is null || !File.Exists(path)) - { - output.WriteLine("LIBRED_V1_PROBE not set to an existing file — nothing to inspect."); - return; - } - - using var db = JetDatabase.Open(path); - output.WriteLine($"{Path.GetFileName(path)}"); - output.WriteLine($" page-0 sort order : lcid {db.DefaultCollationLcid}, version {db.DefaultCollationVersion}"); - foreach (var table in db.Catalog.Tables.Where(t => t.Columns.Any(c => c.Type is JetDataType.Text or JetDataType.Memo))) - { - var collations = table.Columns - .Where(c => c.Type is JetDataType.Text or JetDataType.Memo) - .Select(c => $"{c.Collation.Order} v{c.Collation.Version}") - .Distinct(); - output.WriteLine($" {table.Name,-24} {string.Join(", ", collations)}"); - } - } - - private static object? CreateDbEngine(out string progId) - { - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - progId = $"DAO.DBEngine.{n}"; - Type? type = Type.GetTypeFromProgID(progId); - if (type is null) continue; - try { return Activator.CreateInstance(type); } - catch (Exception) { /* registered but not instantiable in this bitness */ } - } - progId = "(none)"; - return null; - } - - private static object? Invoke(object target, string member, params object?[] args) => - target.GetType().InvokeMember(member, BindingFlags.InvokeMethod, null, target, args); -} diff --git a/test/LibRed.Core.AccessTests/DaoLocaleCollationProbeTest.cs b/test/LibRed.Core.AccessTests/DaoLocaleCollationProbeTest.cs index 0815a00a7..244f8d78f 100644 --- a/test/LibRed.Core.AccessTests/DaoLocaleCollationProbeTest.cs +++ b/test/LibRed.Core.AccessTests/DaoLocaleCollationProbeTest.cs @@ -16,8 +16,13 @@ namespace LibRed.Core.Tests; // directions, and it is worth knowing which of DAO's names still do anything. // // For each: does DAO accept the locale, what LCID lands on disk, will ACE open the result at all, and do the -// index keys actually differ from General v0? DAO can only author version 0 (DaoDatabaseCreationProbeTest), -// so any difference here is a locale difference, not a sort-order-version one. +// index keys actually differ from General v0? DAO can only author version 0 (measured by +// DaoDatabaseCreationProbeTest, now in git history), so any difference here is a locale difference, not a +// sort-order-version one. +// +// Collation.cs cites this as the evidence that the five are inert. A survey rather than a test: it runs only +// with LIBRED_ACE_SURVEYS=1 (AceSurveys). +[Collection(AceCollection.Name)] public class DaoLocaleCollationProbeTest(ITestOutputHelper output) { private const int UseJet = 2; @@ -74,6 +79,7 @@ private static readonly (string Label, string Locale, int ExpectedLcid)[] Locale [Fact] public void Probe_dao_only_collating_orders() { + AceSurveys.RequireOptIn(); object? engine = CreateDbEngine(out string progId); if (engine is null) { output.WriteLine("DAO unavailable in this process."); return; } output.WriteLine($"DAO engine: {progId}"); @@ -202,6 +208,7 @@ private static void Exec(System.Data.OleDb.OleDbConnection connection, string sq private static object? CreateDbEngine(out string progId) { + AceTestDatabase.ReleaseAbandonedComObjects(); foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) { progId = $"DAO.DBEngine.{n}"; diff --git a/test/LibRed.Core.AccessTests/DaoPageLayoutProbeTest.cs b/test/LibRed.Core.AccessTests/DaoPageLayoutProbeTest.cs deleted file mode 100644 index 57287534c..000000000 --- a/test/LibRed.Core.AccessTests/DaoPageLayoutProbeTest.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System.Buffers.Binary; -using LibRed; -using LibRed.Catalog; -using LibRed.Pages; -using Xunit; - -namespace LibRed.Core.Tests; - -// PROBE: what the real engine puts on every page of a freshly created database, and in what order — the page -// budget behind a DAO-created ACE 12 file. Useful for judging how closely LibRed's own bootstrap should -// follow it, and for reading a hex dump of one without guessing. -// -// Every page is labelled from its own header (type byte, and the owning TDEF for data/index pages) and -// cross-referenced against the catalog: each table's TDEF page, its index roots, and the usage-map pages its -// TDEF points at. -public class DaoPageLayoutProbeTest(ITestOutputHelper output) -{ - [Fact] - public void Probe_dao_created_page_layout() - { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } - if (engine is null) { output.WriteLine("DAO unavailable."); return; } - - string path = TemporaryDatabase.CreatePath("dao-layout-"); - File.Delete(path); - try - { - object workspace = Invoke(engine, "CreateWorkspace", "", "admin", "", 2)!; - object database = Invoke(workspace, "CreateDatabase", path, ";LANGID=0x0409;CP=1252;COUNTRY=0", 128)!; - Invoke(database, "Close"); - - byte[] file = File.ReadAllBytes(path); - using var db = JetDatabase.Open(path); - int pageSize = db.Format.PageSize; - int pages = file.Length / pageSize; - - // Label what the catalog knows about: TDEF pages and index roots. - var owners = new Dictionary(); - var labels = new Dictionary(); - foreach (TableDef t in db.Catalog.Tables) - { - owners[t.DefinitionPage] = t.Name; - labels[t.DefinitionPage] = $"TDEF {t.Name}"; - foreach (IndexDef i in t.Indexes) - if (i.RootPage > 0) labels[i.RootPage] = $"index root {t.Name}.{i.Name}"; - } - - output.WriteLine($"{pages} pages of {pageSize} bytes ({file.Length:N0} bytes)"); - output.WriteLine(""); - output.WriteLine("page type owner label"); - for (int p = 0; p < pages; p++) - { - ReadOnlySpan page = file.AsSpan(p * pageSize, pageSize); - var type = (PageType)page[0]; - // Data, index and usage-map pages carry the owning TDEF page at offset 4. - int owner = type is PageType.DataPage or PageType.IntermediateIndexPage - or PageType.LeafIndexPage or PageType.PageUsageBitmap - ? BinaryPrimitives.ReadInt32LittleEndian(page[4..]) - : 0; - string ownerName = owner > 0 && owners.TryGetValue(owner, out string? n) ? $"{owner} {n}" : owner > 0 ? owner.ToString() : ""; - labels.TryGetValue(p, out string? label); - if (label is null && p == 0) label = "database definition"; - if (label is null && p == 1) label = "global free-pages map"; - output.WriteLine($"{p,4} {type,-22} {ownerName,-24} {label}"); - } - - // Which pages does each TDEF name as its usage maps? Those are the "unlabelled" data pages. - output.WriteLine(""); - output.WriteLine("usage-map pointers held in each TDEF (row:page):"); - foreach (TableDef t in db.Catalog.Tables.OrderBy(t => t.DefinitionPage)) - { - ReadOnlySpan tdef = file.AsSpan(t.DefinitionPage * pageSize, pageSize); - // Each pointer is a 1-byte row index then a 3-byte page number. - int owned = BinaryPrimitives.ReadInt32LittleEndian(tdef[db.Format.TdefOwnedPagesOffset..]); - int free = BinaryPrimitives.ReadInt32LittleEndian(tdef[db.Format.TdefFreePagesOffset..]); - output.WriteLine($" {t.Name,-30} owned=page {owned >> 8} row {owned & 0xFF} free=page {free >> 8} row {free & 0xFF}"); - } - } - finally { TemporaryDatabase.Delete(path); } - } - - private static object? Invoke(object target, string member, params object?[] args) => - target.GetType().InvokeMember(member, System.Reflection.BindingFlags.InvokeMethod, null, target, args); -} diff --git a/test/LibRed.Core.AccessTests/DatabaseEncryptionTests.cs b/test/LibRed.Core.AccessTests/DatabaseEncryptionTests.cs index d052026b5..7a660b4cc 100644 --- a/test/LibRed.Core.AccessTests/DatabaseEncryptionTests.cs +++ b/test/LibRed.Core.AccessTests/DatabaseEncryptionTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class DatabaseEncryptionTests { // Resolve against the test assembly's output (where the csproj copies Data\*.accdb), not a hardcoded diff --git a/test/LibRed.Core.AccessTests/DateTime2KeyEncodingTests.cs b/test/LibRed.Core.AccessTests/DateTime2KeyEncodingTests.cs index 11e184e29..8983a9ba1 100644 --- a/test/LibRed.Core.AccessTests/DateTime2KeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/DateTime2KeyEncodingTests.cs @@ -16,6 +16,7 @@ namespace LibRed.Core.Tests; // // The fixture is Northwind (ACE 12) raised to version byte 0x06, which is the entire upgrade - see // AceDateTime2UpgradeTests, which proves ACE asks for nothing more. +[Collection(AceCollection.Name)] public class DateTime2KeyEncodingTests { // Spread across the range, and deliberately including a January date: ACE's own OLE DB reader cannot diff --git a/test/LibRed.Core.AccessTests/DateTimeDefaultAccessTests.cs b/test/LibRed.Core.AccessTests/DateTimeDefaultAccessTests.cs index 4a3efb44c..5fc48a3f8 100644 --- a/test/LibRed.Core.AccessTests/DateTimeDefaultAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DateTimeDefaultAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // Byte-faithful check: a LibRed-written DATETIME column with a NOW() default is read by Access, which opens the // file without repair and applies the default itself on a bare insert (a current timestamp). +[Collection(AceCollection.Name)] public class DateTimeDefaultAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DecimalKeyEncodingTests.cs b/test/LibRed.Core.AccessTests/DecimalKeyEncodingTests.cs index 94c76354b..4e0299de8 100644 --- a/test/LibRed.Core.AccessTests/DecimalKeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/DecimalKeyEncodingTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: a FixedPoint (Numeric/Decimal) index key is a sign byte plus the 16-byte big-endian unscaled // magnitude (|value| * 10^scale). Non-negative uses sign 0xFF; a negative value is the bitwise complement of // the whole 17-byte positive form. Verified against keys Access itself wrote. +[Collection(AceCollection.Name)] public class DecimalKeyEncodingTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DefinitionRewriteParityAccessTests.cs b/test/LibRed.Core.AccessTests/DefinitionRewriteParityAccessTests.cs new file mode 100644 index 000000000..bb41db312 --- /dev/null +++ b/test/LibRed.Core.AccessTests/DefinitionRewriteParityAccessTests.cs @@ -0,0 +1,108 @@ +using System.Data.OleDb; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// Rewriting a table definition through ACE and through LibRed, on copies of the same ACE-built file, must leave the +/// same file (docs/format/page-02a-tdef.md §3.2): the first page rewritten in place — alone, only the 8-byte reserve +/// past the new end zeroed — and continuation data moved to fresh pages, the last allocated first from the lowest +/// free pages, with the old continuation pages released untouched. The file has free pages mid-file, left by a +/// dropped table, so the allocation order shows. +/// +[Collection(AceCollection.Name)] +public class DefinitionRewriteParityAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + [Theory] + [InlineData("a single page shrinking", "S", "DROP COLUMN c001")] + [InlineData("a single page growing onto a second", "N", "ADD COLUMN z1")] + [InlineData("two pages falling back to one", "M", "DROP COLUMN c001")] + [InlineData("two pages growing", "W", "ADD COLUMN z1")] + [InlineData("three pages shrinking", "V", "DROP COLUMN c001")] + [InlineData("three pages growing by an index", "V", "CREATE INDEX")] + public void Libred_rewrites_a_definition_byte_for_byte_with_ace(string label, string table, string rewrite) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "tdefrw-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + Exec(connection, $"CREATE TABLE S (Id LONG{Columns(10, "LONG")}, X TEXT(10))"); + Exec(connection, $"CREATE TABLE N (Id LONG{Columns(114, "LONG")})"); + Exec(connection, "CREATE TABLE Filler (Id LONG, T TEXT(255))"); + for (int i = 0; i < 60; i++) Exec(connection, $"INSERT INTO Filler (Id, T) VALUES ({i}, '{new string('f', 250)}')"); + Exec(connection, $"CREATE TABLE M (Id LONG{Columns(115, "LONG")})"); + Exec(connection, $"CREATE TABLE W (Id LONG{Columns(199, "CURRENCY")})"); + Exec(connection, $"CREATE TABLE V (Id LONG{Columns(254, "CURRENCY")})"); + } + using (OleDbConnection connection = AceTestDatabase.Open(start)) + Exec(connection, "DROP TABLE Filler"); + + string ace = TemporaryDatabase.CopyPath(start, "tdefrw-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, rewrite switch + { + "CREATE INDEX" => $"CREATE INDEX ix ON {table} (c001)", + "ADD COLUMN z1" => $"ALTER TABLE {table} ADD COLUMN z1 LONG", + _ => $"ALTER TABLE {table} {rewrite}", + }); + + string libred = TemporaryDatabase.CopyPath(start, "tdefrw-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + switch (rewrite) + { + case "CREATE INDEX": db.CreateIndex(table, "ix", [("c001", false)]); break; + case "ADD COLUMN z1": Assert.True(db.AddColumn(table, new ColumnSpec("z1", JetDataType.Int32, 4, IsFixedLength: true))); break; + default: Assert.True(db.DropColumn(table, "c001")); break; + } + + output.WriteLine(label); + string difference = DropTableParityAccessTests.Difference(ace, libred); + output.WriteLine(difference); + Assert.Equal("", difference); + } + + // A relationship adds an incoming block to the parent's definition. Into a single page after a drop, the 8 + // reserve bytes past the new end land on the old tail and are zeroed, the rest left; into a multi-page parent, + // the continuation data moves to fresh pages. The relationship's own type-8 MSysObjects object and its two + // MSysACEs rows are written as ACE writes them. + [Theory] + [InlineData("a shrunk single-page parent", false)] + [InlineData("a multi-page parent", true)] + public void Libred_adds_a_relationship_into_the_parents_definition_byte_for_byte_with_ace(string label, bool wide) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "tdefrel-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + Exec(connection, wide + ? $"CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY{Columns(199, "CURRENCY")})" + : "CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY, abcdefghijklmnopqrst LONG, V LONG)"); + Exec(connection, "CREATE TABLE C (Id LONG, PId LONG)"); + } + if (!wide) + using (OleDbConnection connection = AceTestDatabase.Open(start)) + Exec(connection, "ALTER TABLE P DROP COLUMN abcdefghijklmnopqrst"); + + string ace = TemporaryDatabase.CopyPath(start, "tdefrel-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, "ALTER TABLE C ADD CONSTRAINT fkCP FOREIGN KEY (PId) REFERENCES P (Id)"); + + string libred = TemporaryDatabase.CopyPath(start, "tdefrel-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + db.AddForeignKey("C", new RelationshipSpec("fkCP", "P", [("PId", "Id")], IsEnforced: true, CascadeUpdate: false, CascadeDelete: false)); + + output.WriteLine(label); + string difference = DropTableParityAccessTests.Difference(ace, libred); + output.WriteLine(difference); + Assert.Equal("", difference); + } + + private static string Columns(int count, string type) => + string.Concat(Enumerable.Range(1, count).Select(i => $", c{i:D3} {type}")); + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/DeleteAccessTests.cs b/test/LibRed.Core.AccessTests/DeleteAccessTests.cs index 0d9d68f85..7213479a6 100644 --- a/test/LibRed.Core.AccessTests/DeleteAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DeleteAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// LibRed soft-deletes a row (slot flagged, index entries removed, TDEF row count decremented) and Access /// reads the table without it — the deleted row is gone from scans, seeks, and COUNT. /// +[Collection(AceCollection.Name)] public class DeleteAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DerivedTableViewAccessTests.cs b/test/LibRed.Core.AccessTests/DerivedTableViewAccessTests.cs index a6c3f02ec..b29df26b7 100644 --- a/test/LibRed.Core.AccessTests/DerivedTableViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DerivedTableViewAccessTests.cs @@ -13,6 +13,7 @@ namespace LibRed.Core.Tests; /// inner subquery SQL in Expression and the alias in Name2 (no Name1). This checks /// LibRed writes it that way and that Access opens the file and runs the view. /// +[Collection(AceCollection.Name)] public class DerivedTableViewAccessTests { private const string Subquery = diff --git a/test/LibRed.Core.AccessTests/DistinctViewAccessTests.cs b/test/LibRed.Core.AccessTests/DistinctViewAccessTests.cs index a819ba6ad..d803428eb 100644 --- a/test/LibRed.Core.AccessTests/DistinctViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DistinctViewAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// Access runs a LibRed-created DISTINCT view with a RIGHT JOIN and a BETWEEN/date WHERE — Northwind's /// "Quarterly Orders". DISTINCT is the MSysQueries flag row; the WHERE is stored verbatim. /// +[Collection(AceCollection.Name)] public class DistinctViewAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropColumnAccessTests.cs b/test/LibRed.Core.AccessTests/DropColumnAccessTests.cs index 7b3542d1b..18ff12e0e 100644 --- a/test/LibRed.Core.AccessTests/DropColumnAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropColumnAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; // keep their original column ids (a gap appears) and their original variable-table index. So a correct // reader must read the STORED variable index (descriptor 0x07), not derive it by ranking column ids — // otherwise a survivor after a dropped variable column decodes the wrong slot. +[Collection(AceCollection.Name)] public class DropColumnAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropColumnConstraintAccessTests.cs b/test/LibRed.Core.AccessTests/DropColumnConstraintAccessTests.cs index da18204fb..2cf0c9278 100644 --- a/test/LibRed.Core.AccessTests/DropColumnConstraintAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropColumnConstraintAccessTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; // Ground truth for our DROP COLUMN guard: ACE REJECTS dropping a column that is part of an index/key or a // relationship — it never cascades, you must drop the dependent first. (Our TableCreator.DropColumn mirrors // this: it throws for an indexed/keyed column and for a column participating in a relationship.) +[Collection(AceCollection.Name)] public class DropColumnConstraintAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropColumnLvPropAccessTests.cs b/test/LibRed.Core.AccessTests/DropColumnLvPropAccessTests.cs index 6b774b21f..c737fcd5f 100644 --- a/test/LibRed.Core.AccessTests/DropColumnLvPropAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropColumnLvPropAccessTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // DROP COLUMN must remove the dropped column's DefaultValue/Required entries from the table's // MSysObjects.LvProp blob — verified this is what ACE does. LibRed's drop does the same (surgical removal // of the column's property block), and ACE still opens/reads the result. +[Collection(AceCollection.Name)] public class DropColumnLvPropAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropColumnParityAccessTests.cs b/test/LibRed.Core.AccessTests/DropColumnParityAccessTests.cs new file mode 100644 index 000000000..e9beeab94 --- /dev/null +++ b/test/LibRed.Core.AccessTests/DropColumnParityAccessTests.cs @@ -0,0 +1,73 @@ +using System.Data.OleDb; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// DROP COLUMN of a memo or OLE column through ACE and through LibRed, on copies of the same ACE-built file, must leave +/// the same file: the column's long-value map entry gone from the definition, its owned and free map records retired +/// from their holder as DROP TABLE retires them, and the pages it owned back in the global free map +/// (docs/format/long-values.md). +/// +[Collection(AceCollection.Name)] +public class DropColumnParityAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + // Each column holds an inline value, a single-page value and a chained one. With full pages, five more values + // each fill half a page, so the column also owns full single-value pages that are no longer in its free map. + [Theory] + [InlineData("M1", false)] + [InlineData("O1", false)] + [InlineData("M1", true)] + [InlineData("O1", true)] + public void Libred_drops_a_long_value_column_byte_for_byte_with_ace(string column, bool fullPages) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "dropcolpar-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + Exec(connection, "CREATE TABLE T (Id LONG, M1 MEMO, X TEXT(10), M2 MEMO, O1 LONGBINARY)"); + Insert(connection, 1, "short", "two", Bytes(10)); + Insert(connection, 2, new string('a', 1000), new string('b', 900), Bytes(1500)); + Insert(connection, 3, new string('c', 10000), "mid", Bytes(9000)); + if (fullPages) + for (int id = 4; id <= 8; id++) + Insert(connection, id, new string('f', 1010), "m", Bytes(2020)); + } + + string ace = TemporaryDatabase.CopyPath(start, "dropcolpar-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, $"ALTER TABLE T DROP COLUMN {column}"); + + string libred = TemporaryDatabase.CopyPath(start, "dropcolpar-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + Assert.True(db.DropColumn("T", column)); + + string difference = DropTableParityAccessTests.Difference(ace, libred); + output.WriteLine(difference); + Assert.Equal("", difference); + + // ACE reads the surviving long values from the file LibRed wrote. + using OleDbConnection check = AceTestDatabase.Open(libred); + using OleDbCommand read = check.CreateCommand(); + read.CommandText = "SELECT M2 FROM T WHERE Id = 2"; + Assert.Equal(new string('b', 900), read.ExecuteScalar()); + } + + private static byte[] Bytes(int n) => Enumerable.Range(0, n).Select(i => (byte)(i * 7 + 1)).ToArray(); + + private static void Insert(OleDbConnection connection, int id, string m1, string m2, byte[] o1) + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = $"INSERT INTO T (Id, M1, X, M2, O1) VALUES ({id}, ?, 'x', ?, ?)"; + insert.Parameters.Add("m1", OleDbType.LongVarWChar).Value = m1; + insert.Parameters.Add("m2", OleDbType.LongVarWChar).Value = m2; + insert.Parameters.Add("o1", OleDbType.LongVarBinary).Value = o1; + insert.ExecuteNonQuery(); + } + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/DropConstraintAccessTests.cs b/test/LibRed.Core.AccessTests/DropConstraintAccessTests.cs index 32dfd6a61..67eb8790d 100644 --- a/test/LibRed.Core.AccessTests/DropConstraintAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropConstraintAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// After LibRed drops a foreign key (soft-deleting its MSysRelationships rows), Access still opens the /// file and reads the tables — the soft delete is a normal Jet delete, not corruption. /// +[Collection(AceCollection.Name)] public class DropConstraintAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropIndexAccessTests.cs b/test/LibRed.Core.AccessTests/DropIndexAccessTests.cs index 0baa2a3c0..95769d585 100644 --- a/test/LibRed.Core.AccessTests/DropIndexAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropIndexAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // ACE's DROP INDEX removes the index's TDEF blocks and frees its B-tree root; it allows dropping plain, // unique and even the primary-key index, but REJECTS an FK-backing index ("used in a relationship"). // LibRed mirrors this, and ACE opens+reads a LibRed-index-dropped file. +[Collection(AceCollection.Name)] public class DropIndexAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropTableAccessTests.cs b/test/LibRed.Core.AccessTests/DropTableAccessTests.cs index 48358d47a..e19a41772 100644 --- a/test/LibRed.Core.AccessTests/DropTableAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropTableAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // ACE's DROP TABLE removes the object's MSysObjects + MSysACEs rows and frees its pages (a later create // reuses them). LibRed mirrors this: after a LibRed DROP TABLE, ACE opens the file, no longer sees the // table, reads the other tables, and reuses the freed pages when creating a new table. +[Collection(AceCollection.Name)] public class DropTableAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DropTableParityAccessTests.cs b/test/LibRed.Core.AccessTests/DropTableParityAccessTests.cs new file mode 100644 index 000000000..ee7c7c28e --- /dev/null +++ b/test/LibRed.Core.AccessTests/DropTableParityAccessTests.cs @@ -0,0 +1,222 @@ +using System.Data.OleDb; +using System.Text; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// DROP TABLE through ACE and through LibRed, on copies of the same ACE-built file, must leave the same file: every +/// page the table owned freed by both — its indexes' B-tree pages, a wide definition's continuation pages and a +/// large map's bitmap pages included — and the map holders retired the same way +/// (docs/format/page-05-usage-maps.md §9, page-08-released-tdef.md). +/// +[Collection(AceCollection.Name)] +public class DropTableParityAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + private const int PageSize = 4096; + + [Theory] + [InlineData("primary key", "CREATE TABLE Doomed (Id LONG CONSTRAINT pk PRIMARY KEY, M MEMO)", 40, 20_000)] + // The second index's map record is below the long-value maps, so it slides as they are retired. + [InlineData("primary key and a secondary index", + "CREATE TABLE Doomed (Id LONG CONSTRAINT pk PRIMARY KEY, M MEMO);CREATE INDEX ixId ON Doomed (Id DESC)", 40, 20_000)] + [InlineData("a multi-page index", "CREATE TABLE Doomed (Id LONG CONSTRAINT pk PRIMARY KEY, M TEXT(200));CREATE INDEX ixM ON Doomed (M)", 3000, 200)] + public void Libred_drops_an_indexed_table_byte_for_byte_with_ace(string label, string create, int rows, int chars) + { + AssertDropMatchesAce(label, connection => + { + foreach (string statement in create.Split(';')) Exec(connection, statement); + for (int i = 1; i <= rows; i++) + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = "INSERT INTO Doomed (Id, M) VALUES (?, ?)"; + insert.Parameters.Add("id", OleDbType.Integer).Value = i; + insert.Parameters.Add("m", OleDbType.VarWChar, chars).Value = + i.ToString("D6") + new string((char)('a' + i % 26), chars - 6); + insert.ExecuteNonQuery(); + } + }); + } + + // 255 columns: the definition runs onto continuation pages. At 32,768 rows of one full page each, the table's + // owned map outgrows an inline record and becomes a reference map with two bitmap pages. + [Theory] + [InlineData("a multi-page definition", 1)] + [InlineData("a multi-page definition and a reference-form owned map", 32_768)] + public void Libred_drops_a_wide_table_byte_for_byte_with_ace(string label, int rows) + { + AssertDropMatchesAce(label, connection => FillWide(connection, "Doomed", rows)); + } + + // How the close sizes the released-pages map when an inline record cannot simply grow from its start page, + // each drop in its own session. W ends between pages 32,000 and 32,735, a filler F follows, and X lies wholly + // past page 32,736 — the second bitmap range. + // - W then X: W's pages, 310..~32,300, are too wide for an inline record, so the first close converts the map + // with a bitmap page for the first range only; X's close then adds one for the second. + // - X alone: X's pages fit an inline record once its window moves to start at X, so the map moves rather than + // converting. + [Fact] + public void Libred_sizes_the_released_map_as_ace_does_across_sessions() + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "droppar-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + FillWide(connection, "W", 31_900); + FillWide(connection, "F", 500); + FillWide(connection, "X", 1_000); + } + + AssertDropsMatchAce("W then X", start, "W", "X"); + AssertDropsMatchAce("X alone", start, "X"); + } + + // Released pages in the first and third bitmap ranges and none in the second: A sits in range 0; C's definition, + // map holder and first row are made next, in range 0 too; B then fills the file past page 65,472; C's other + // rows land in range 2. Dropping A and C in one session converts the released map with bitmap pages for ranges + // 0 and 2 only, and grows the inline record first only as far as the highest released page it reaches, not to + // its longest length. Explicit: the file is ~272 MB, built once and copied twice, and takes minutes. + [Fact(Explicit = true)] + public void Libred_converts_the_released_map_without_a_bitmap_page_for_an_empty_range() + { + const int Range2 = 2 * 32_736; + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "droppar-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) FillWide(connection, "A", 700); + using (OleDbConnection connection = AceTestDatabase.Open(start)) FillWide(connection, "C", 1); + using (OleDbConnection connection = AceTestDatabase.Open(start)) FillWide(connection, "B", 1); + for (int rows = 1; new FileInfo(start).Length / PageSize < Range2 + 40;) + { + int add = (int)Math.Clamp(Range2 + 40 - new FileInfo(start).Length / PageSize, 64, 32_000); + using (OleDbConnection connection = AceTestDatabase.Open(start)) AppendWide(connection, "B", rows, add); + rows += add; + } + using (OleDbConnection connection = AceTestDatabase.Open(start)) AppendWide(connection, "C", 1, 999); + + string ace = TemporaryDatabase.CopyPath(start, "droppar-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + { + Exec(connection, "DROP TABLE A"); + Exec(connection, "DROP TABLE C"); + } + + string libred = TemporaryDatabase.CopyPath(start, "droppar-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + { + Assert.True(db.DropTable("A")); + Assert.True(db.DropTable("C")); + } + + output.WriteLine($"comparing {new FileInfo(ace).Length / PageSize} pages"); + string difference = Difference(ace, libred); + output.WriteLine(difference); + Assert.Equal("", difference); + } + + /// A 255-column table — its definition runs onto continuation pages — of + /// rows of one full page each, filled by repeatedly copying the rows already there. + private static void FillWide(OleDbConnection connection, string table, int rows) + { + var columns = new StringBuilder($"Id LONG CONSTRAINT pk{table} PRIMARY KEY"); + var values = new StringBuilder("0"); + for (int i = 1; i < 255; i++) + { + columns.Append($", c{i} CURRENCY"); + values.Append($", {i}"); + } + Exec(connection, $"CREATE TABLE {table} ({columns})"); + Exec(connection, $"INSERT INTO {table} ({WideNames()}) VALUES ({values})"); + AppendWide(connection, table, 1, rows - 1); + } + + /// Adds rows to a table holding , + /// copying the first rows with their ids moved past the last. + private static void AppendWide(OleDbConnection connection, string table, int have, int add) + { + string names = WideNames(); + string rest = names["Id".Length..]; + while (add > 0) + { + int step = Math.Min(Math.Min(have, add), 16_000); + Exec(connection, $"INSERT INTO {table} ({names}) SELECT TOP {step} Id + {have}{rest} FROM {table} ORDER BY Id"); + have += step; + add -= step; + } + } + + private static string WideNames() + { + var names = new StringBuilder("Id"); + for (int i = 1; i < 255; i++) names.Append($", c{i}"); + return names.ToString(); + } + + private void AssertDropMatchesAce(string label, Action build) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "droppar-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + build(connection); + AssertDropsMatchAce(label, start, "Doomed"); + } + + /// Drops from copies of through each engine, one + /// session per drop, and compares the results. + private void AssertDropsMatchAce(string label, string start, params string[] tables) + { + string ace = TemporaryDatabase.CopyPath(start, "droppar-ace-"); + foreach (string table in tables) + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, $"DROP TABLE {table}"); + + string libred = TemporaryDatabase.CopyPath(start, "droppar-lib-"); + foreach (string table in tables) + using (var db = JetDatabase.Open(libred, readOnly: false)) + Assert.True(db.DropTable(table)); + + output.WriteLine($"{label}: comparing {new FileInfo(ace).Length / PageSize} pages"); + string difference = Difference(ace, libred); + output.WriteLine(difference); + Assert.Equal("", difference); + } + + /// Every differing byte of every page, except page 0 (the modification counter), MSysObjects' data + /// page (owner 2, the DateUpdate wall clock) and index pages — removing the catalog rows leaves the two + /// engines with identical index content on byte-different pages, accepted in page-03-04 §10.4a. The dropped + /// table's own index pages are compared through the free map instead. + internal static string Difference(string acePath, string libredPath) + { + byte[] ace = File.ReadAllBytes(acePath), libred = File.ReadAllBytes(libredPath); + var differences = new StringBuilder(); + int pages = Math.Max(ace.Length, libred.Length) / PageSize, lines = 0; + for (int page = 1; page < pages && lines < 200; page++) + { + int at = page * PageSize; + bool inAce = at + PageSize <= ace.Length, inLibRed = at + PageSize <= libred.Length; + if (!inAce || !inLibRed) + { + differences.AppendLine($"page {page}: present in {(inAce ? "ACE" : "LibRed")} only"); + lines++; + continue; + } + if (BitConverter.ToInt32(ace, at + 4) == 2) continue; + if (ace[at] is (byte)LibRed.Pages.PageType.IntermediateIndexPage or (byte)LibRed.Pages.PageType.LeafIndexPage) continue; + + for (int i = 0, shown = 0; i < PageSize && shown < 8; i++) + if (ace[at + i] != libred[at + i]) + { + differences.AppendLine( + $"page {page} (type 0x{ace[at]:X2} owner {BitConverter.ToInt32(ace, at + 4)}) " + + $"+0x{i:X3}: ace={ace[at + i]:X2} libred={libred[at + i]:X2}"); + shown++; + lines++; + } + } + return differences.ToString(); + } + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.CommandTimeout = 0; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/DropViewAccessTests.cs b/test/LibRed.Core.AccessTests/DropViewAccessTests.cs index c2e13ef55..2dea5105a 100644 --- a/test/LibRed.Core.AccessTests/DropViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/DropViewAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; // DROP VIEW / DROP PROCEDURE removes a type-5 query object's MSysObjects + MSysQueries + MSysACEs rows // (verified vs ACE, which also treats the two statements interchangeably). After a LibRed drop, ACE opens // the file, no longer sees the object, and still runs the surviving views. +[Collection(AceCollection.Name)] public class DropViewAccessTests { private static OleDbConnection Open(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/DuplicateIndexKeyProbeTest.cs b/test/LibRed.Core.AccessTests/DuplicateIndexKeyProbeTest.cs index b5c7cbd41..7b4c7436d 100644 --- a/test/LibRed.Core.AccessTests/DuplicateIndexKeyProbeTest.cs +++ b/test/LibRed.Core.AccessTests/DuplicateIndexKeyProbeTest.cs @@ -14,6 +14,7 @@ namespace LibRed.Core.Tests; // // The cause was that the shared prefix covers the whole entry, trailer included — see IndexPageReader — so // the stored remainder can be two bytes. These cases now assert, since nothing about them is exotic. +[Collection(AceCollection.Name)] public class DuplicateIndexKeyProbeTest(ITestOutputHelper output) { [Theory] diff --git a/test/LibRed.Core.AccessTests/FixedCharEncodingTests.cs b/test/LibRed.Core.AccessTests/FixedCharEncodingTests.cs index 23b143189..41e203e2f 100644 --- a/test/LibRed.Core.AccessTests/FixedCharEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/FixedCharEncodingTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // A fixed-length text (CHAR/NCHAR) column stores space-padded to its full width — matching ACE. Previously // LibRed's encoder didn't pad, so inserting a short value into a fixed CHAR threw // "Column ... encoded to N bytes, expected M". This covers the round-trip and the ACE read-back. +[Collection(AceCollection.Name)] public class FixedCharEncodingTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/FixedFlagHonouredAccessTests.cs b/test/LibRed.Core.AccessTests/FixedFlagHonouredAccessTests.cs index d51e72f2e..b594f2592 100644 --- a/test/LibRed.Core.AccessTests/FixedFlagHonouredAccessTests.cs +++ b/test/LibRed.Core.AccessTests/FixedFlagHonouredAccessTests.cs @@ -29,6 +29,7 @@ namespace LibRed.Core.Tests; // the 2016-era build, and Large Number arrived in a later servicing build - so "needs ACE 16" is about the // Access version, not about anything named 2016. Hence the skip: an engine that cannot create a BIGINT // cannot open a file holding one, and that says nothing about the descriptor behaviour under test. +[Collection(AceCollection.Name)] public class FixedFlagHonouredAccessTests : TempDatabaseTest { public static TheoryData Shapes => new() diff --git a/test/LibRed.Core.AccessTests/FixedWidthOverflowAccessTests.cs b/test/LibRed.Core.AccessTests/FixedWidthOverflowAccessTests.cs index a72f06c58..dde900c40 100644 --- a/test/LibRed.Core.AccessTests/FixedWidthOverflowAccessTests.cs +++ b/test/LibRed.Core.AccessTests/FixedWidthOverflowAccessTests.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // // The CHAR/BINARY half also establishes that the fixed form is reachable from ordinary ACE DDL — it is not a // LibRed-only construct that only the Core API can produce. +[Collection(AceCollection.Name)] public class FixedWidthOverflowAccessTests { [Fact] diff --git a/test/LibRed.Core.AccessTests/GeneralV1CollationAccessTests.cs b/test/LibRed.Core.AccessTests/GeneralV1CollationAccessTests.cs index b0d397d79..1cc944176 100644 --- a/test/LibRed.Core.AccessTests/GeneralV1CollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/GeneralV1CollationAccessTests.cs @@ -13,6 +13,7 @@ namespace LibRed.Core.Tests; // is also the point: nothing else here can make one. DAO always writes v0, ignoring the application setting, // and Access only honours "New database sort order" through its own UI. So this doubles as the test that // LibRed's create-with-collation produces a file ACE accepts as a General database. +[Collection(AceCollection.Name)] public class GeneralV1CollationAccessTests { private static readonly string[] Samples = diff --git a/test/LibRed.Core.AccessTests/GlobalMapPointerAccessTests.cs b/test/LibRed.Core.AccessTests/GlobalMapPointerAccessTests.cs new file mode 100644 index 000000000..1b4ae35c4 --- /dev/null +++ b/test/LibRed.Core.AccessTests/GlobalMapPointerAccessTests.cs @@ -0,0 +1,130 @@ +using System.Buffers.Binary; +using System.Data.OleDb; +using LibRed.Catalog; +using LibRed.Formats; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +// LibRed allocates through page 0's global map pointers (0x18 free, 0x1C released), as ACE does. Two files ACE +// itself reads correctly — maps moved off page 1, and pages sitting in the released map — must stay files ACE +// reads correctly after LibRed has allocated into them. +[Collection(AceCollection.Name)] +public class GlobalMapPointerAccessTests : TempDatabaseTest +{ + private const int NorthwindPages = 353; + + [Fact] + public void Ace_reads_a_table_libred_filled_through_maps_moved_off_page_one() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-ace-moved-"); + byte[] page1 = ReadPage(path, 1); + byte[] moved = (byte[])page1.Clone(); + SetMapBit(moved, row: 0, page: NorthwindPages, set: false); + AppendPage(path, moved); + WritePointer(path, JetFormatBase.FreePagesMapPointerOffset, row: 0, page: NorthwindPages); + WritePointer(path, JetFormatBase.ReleasedPagesMapPointerOffset, row: 1, page: NorthwindPages); + + FillWithLibRed(path, rows: 1500); + + Assert.Equal(page1, ReadPage(path, 1)); // LibRed never touched the stale maps + AssertAceReads(path, rows: 1500); + } + + [Fact] + public void Ace_reads_a_table_libred_filled_around_released_pages() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-ace-released-"); + byte[] page1 = ReadPage(path, 1); + foreach (int p in new[] { 310, 329, NorthwindPages, NorthwindPages + 1 }) + SetMapBit(page1, row: 1, page: p, set: true); + WritePage(path, 1, page1); + + FillWithLibRed(path, rows: 1500); + + // Never allocated while open; the close merged them into the free map and cleared the released map. + byte[] after = ReadPage(path, 1); + foreach (int p in new[] { 310, 329, NorthwindPages, NorthwindPages + 1 }) + { + Assert.False(MapBit(after, 1, p), $"page {p} should no longer be released"); + Assert.True(MapBit(after, 0, p), $"page {p} should be free"); + } + AssertAceReads(path, rows: 1500); + } + + private static void FillWithLibRed(string path, int rows) + { + using var db = JetDatabase.Open(path, readOnly: false); + db.CreateTable("Filled", [ + new ColumnSpec("K", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("V", JetDataType.Text, 510, IsFixedLength: false), // TEXT(255): the length is in bytes + ]); + Table table = db.OpenTable("Filled"); + string value = new('v', 255); + for (int i = 0; i < rows; i++) table.Insert([i, value]); + } + + private static void AssertAceReads(string path, int rows) + { + using OleDbConnection connection = AceTestDatabase.Open(path); + using OleDbCommand count = connection.CreateCommand(); + count.CommandText = "SELECT COUNT(*), SUM(K) FROM Filled"; + using OleDbDataReader reader = count.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(rows, Convert.ToInt32(reader.GetValue(0))); + Assert.Equal((long)rows * (rows - 1) / 2, Convert.ToInt64(reader.GetValue(1))); + } + + private static void WritePointer(string path, int offset, int row, int page) + { + ReadOnlySpan mask = JetFormatBase.PageZeroHeaderMask; + byte[] value = BitConverter.GetBytes((uint)(page << 8 | row)); + for (int i = 0; i < 4; i++) value[i] ^= mask[offset - JetFormatBase.PageZeroHeaderMaskStart + i]; + using var s = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + s.Position = offset; + s.Write(value); + } + + private static byte[] ReadPage(string path, int page) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + var bytes = new byte[4096]; + s.Position = page * 4096L; + s.ReadExactly(bytes); + return bytes; + } + + private static void WritePage(string path, int page, byte[] bytes) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + s.Position = page * 4096L; + s.Write(bytes); + } + + private static void AppendPage(string path, byte[] bytes) + { + using var s = new FileStream(path, FileMode.Append, FileAccess.Write); + s.Write(bytes); + } + + private static (int Byte, int Bit) Locate(byte[] holder, int row, int page) + { + int offset = BinaryPrimitives.ReadUInt16LittleEndian(holder.AsSpan(14 + row * 2)) & 0x1FFF; + int bit = page - BinaryPrimitives.ReadInt32LittleEndian(holder.AsSpan(offset + 1)); + return (offset + 5 + bit / 8, bit % 8); + } + + private static void SetMapBit(byte[] holder, int row, int page, bool set) + { + (int b, int bit) = Locate(holder, row, page); + if (set) holder[b] |= (byte)(1 << bit); + else holder[b] &= (byte)~(1 << bit); + } + + private static bool MapBit(byte[] holder, int row, int page) + { + (int b, int bit) = Locate(holder, row, page); + return (holder[b] & (1 << bit)) != 0; + } +} diff --git a/test/LibRed.Core.AccessTests/GroupByViewAccessTests.cs b/test/LibRed.Core.AccessTests/GroupByViewAccessTests.cs index 5ef9fc727..1b2d7d866 100644 --- a/test/LibRed.Core.AccessTests/GroupByViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/GroupByViewAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// A "totals" (GROUP BY) view — Northwind's "Order Subtotals". Access stores each GROUP BY column as an /// Attribute=9 row; aggregate output columns are ordinary Attribute=6 rows. Access runs it. /// +[Collection(AceCollection.Name)] public class GroupByViewAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/GuidColumnStorageAccessTests.cs b/test/LibRed.Core.AccessTests/GuidColumnStorageAccessTests.cs index 58201022c..c311fa383 100644 --- a/test/LibRed.Core.AccessTests/GuidColumnStorageAccessTests.cs +++ b/test/LibRed.Core.AccessTests/GuidColumnStorageAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // spend record budget ACE does not spend, so a 252-GUID table ACE creates without complaint used to exceed // LibRed's declared-record limit (ColumnWidthLimitAccessTests). The mapping itself lives in the engine, so // LibRed's own side of this is asserted in LibRed.Engine.AccessTests. +[Collection(AceCollection.Name)] public class GuidColumnStorageAccessTests : TempDatabaseTest { [Theory] diff --git a/test/LibRed.Core.AccessTests/GuidKeyEncodingTests.cs b/test/LibRed.Core.AccessTests/GuidKeyEncodingTests.cs index ce124c330..e30fa17ad 100644 --- a/test/LibRed.Core.AccessTests/GuidKeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/GuidKeyEncodingTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class GuidKeyEncodingTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/IndexColumnCountAccessTests.cs b/test/LibRed.Core.AccessTests/IndexColumnCountAccessTests.cs index a4e59a343..b6673bd51 100644 --- a/test/LibRed.Core.AccessTests/IndexColumnCountAccessTests.cs +++ b/test/LibRed.Core.AccessTests/IndexColumnCountAccessTests.cs @@ -18,6 +18,7 @@ namespace LibRed.Core.Tests; // different columns from the ones requested. It was also internally inconsistent: EnsureNoDuplicateKeys // validated using all eleven requested columns while BackfillIndex populated the index from the ten the // TDEF actually recorded. +[Collection(AceCollection.Name)] public class IndexColumnCountAccessTests : TempDatabaseTest { private const int Columns = 12; diff --git a/test/LibRed.Core.AccessTests/IndexKeyTruncationAccessTests.cs b/test/LibRed.Core.AccessTests/IndexKeyTruncationAccessTests.cs index b9e3fbcd1..9592f48a2 100644 --- a/test/LibRed.Core.AccessTests/IndexKeyTruncationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/IndexKeyTruncationAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // data byte, and every measurement behind the rule happened to use an all-text key. These tests pin down both // halves against ACE, because a wrong key here is the silent kind of wrong: neither engine errors, ACE writes // its own key into the same index, and seeks quietly miss rows. +[Collection(AceCollection.Name)] public class IndexKeyTruncationAccessTests(ITestOutputHelper output) { [Fact] diff --git a/test/LibRed.Core.AccessTests/IndexMaintenanceAccessTests.cs b/test/LibRed.Core.AccessTests/IndexMaintenanceAccessTests.cs index 5bd2a0752..4c9b519b3 100644 --- a/test/LibRed.Core.AccessTests/IndexMaintenanceAccessTests.cs +++ b/test/LibRed.Core.AccessTests/IndexMaintenanceAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// Cross-engine update/delete coverage for split indexes and relocated indexed rows. +[Collection(AceCollection.Name)] public class IndexMaintenanceAccessTests { private const int RowCount = 900; diff --git a/test/LibRed.Core.AccessTests/IndexOrderingAccessTests.cs b/test/LibRed.Core.AccessTests/IndexOrderingAccessTests.cs index caa4ff89d..83fd24488 100644 --- a/test/LibRed.Core.AccessTests/IndexOrderingAccessTests.cs +++ b/test/LibRed.Core.AccessTests/IndexOrderingAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// Checks boundary-value index byte encoding and traversal order against ACE's actual ORDER BY. +[Collection(AceCollection.Name)] public class IndexOrderingAccessTests { public static TheoryData KeyFamilies => new() diff --git a/test/LibRed.Core.AccessTests/IndexSplitAccessTests.cs b/test/LibRed.Core.AccessTests/IndexSplitAccessTests.cs index 531f23a34..7e993cdbc 100644 --- a/test/LibRed.Core.AccessTests/IndexSplitAccessTests.cs +++ b/test/LibRed.Core.AccessTests/IndexSplitAccessTests.cs @@ -11,6 +11,7 @@ namespace LibRed.Core.Tests; /// node), Access opens the file and resolves indexed point seeks, an indexed range, a full table scan, /// and a leaf-chain COUNT(*)/SUM — all reaching every row. /// +[Collection(AceCollection.Name)] public class IndexSplitAccessTests { private const int N = 1200; // well past one leaf, so the PK B-tree splits and the root grows a level diff --git a/test/LibRed.Core.AccessTests/IndexUsageMapTests.cs b/test/LibRed.Core.AccessTests/IndexUsageMapTests.cs index 763554b59..3f5213cb6 100644 --- a/test/LibRed.Core.AccessTests/IndexUsageMapTests.cs +++ b/test/LibRed.Core.AccessTests/IndexUsageMapTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; /// Verified against ACE: the union of a table's index maps equals exactly the set of index pages ACE itself /// marks, and LibRed reproduces that. /// +[Collection(AceCollection.Name)] public class IndexUsageMapTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/KeyChecksumProbeTest.cs b/test/LibRed.Core.AccessTests/KeyChecksumProbeTest.cs index 266463895..fdc90e4d7 100644 --- a/test/LibRed.Core.AccessTests/KeyChecksumProbeTest.cs +++ b/test/LibRed.Core.AccessTests/KeyChecksumProbeTest.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; // plain truncation. Not knowing it is the only reason LibRed refuses those values instead of matching them. // // Opt-in via LIBRED_CHECKSUM=1. +[Collection(AceCollection.Name)] public class KeyChecksumProbeTest(ITestOutputHelper output) { /// diff --git a/test/LibRed.Core.AccessTests/Latin1SymbolCollationAccessTests.cs b/test/LibRed.Core.AccessTests/Latin1SymbolCollationAccessTests.cs index 0d8bd679f..f3ab852d4 100644 --- a/test/LibRed.Core.AccessTests/Latin1SymbolCollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/Latin1SymbolCollationAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // refuse them outright ("collation weight is not implemented yet"), so a text column holding "£100" or "Café ©" // could not be indexed. ACE is the oracle here — it builds the index, LibRed re-encodes the same value and must // reproduce the stored bytes exactly. +[Collection(AceCollection.Name)] public class Latin1SymbolCollationAccessTests { // Every printable Latin-1 character outside A–Z/a–z/0–9 and the ASCII punctuation LibRed already knew, diff --git a/test/LibRed.Core.AccessTests/LibRed.Core.AccessTests.csproj b/test/LibRed.Core.AccessTests/LibRed.Core.AccessTests.csproj index 8de06a2bf..a6a357b75 100644 --- a/test/LibRed.Core.AccessTests/LibRed.Core.AccessTests.csproj +++ b/test/LibRed.Core.AccessTests/LibRed.Core.AccessTests.csproj @@ -45,12 +45,6 @@ - - - PreserveNewest - - - diff --git a/test/LibRed.Core.AccessTests/LocaleCollationAccessTests.cs b/test/LibRed.Core.AccessTests/LocaleCollationAccessTests.cs index cd70684ae..69609074a 100644 --- a/test/LibRed.Core.AccessTests/LocaleCollationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/LocaleCollationAccessTests.cs @@ -16,6 +16,7 @@ namespace LibRed.Core.Tests; // The sample set is deliberately much wider than the tailoring: the whole ASCII range, every Latin-1 and // Latin Extended-A letter, and words. A tailoring is only trustworthy if it is also correct for the // characters it does *not* mention. +[Collection(AceCollection.Name)] public class LocaleCollationAccessTests(ITestOutputHelper output) { public static TheoryData Fixtures() => diff --git a/test/LibRed.Core.AccessTests/LocaleFixtureCollationProbeTest.cs b/test/LibRed.Core.AccessTests/LocaleFixtureCollationProbeTest.cs index e92665f71..92b8e4227 100644 --- a/test/LibRed.Core.AccessTests/LocaleFixtureCollationProbeTest.cs +++ b/test/LibRed.Core.AccessTests/LocaleFixtureCollationProbeTest.cs @@ -19,6 +19,7 @@ namespace LibRed.Core.Tests; // 2. Does the v1 scheme generalise beyond General? Romanian and Croatian are the only Latin-script orders // Access offers in "- Legacy" / current pairs, so they are the cheapest test of whether a non-General // order ever reaches sort-order version 1 and 2-byte NLS primaries. +[Collection(AceCollection.Name)] public class LocaleFixtureCollationProbeTest(ITestOutputHelper output) { /// Everything in Data\ that is a fixture for something else. Anything else is treated as a diff --git a/test/LibRed.Core.AccessTests/LongTextStorageAccessTests.cs b/test/LibRed.Core.AccessTests/LongTextStorageAccessTests.cs index 34a0d5e40..0fe061145 100644 --- a/test/LibRed.Core.AccessTests/LongTextStorageAccessTests.cs +++ b/test/LibRed.Core.AccessTests/LongTextStorageAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // // Scope: this is the SQL-created column. A column Access's designer creates with Unicode Compression set // to Yes carries the flag, and LibRed reads that form back (the all-compressed case) but never writes it. +[Collection(AceCollection.Name)] public class LongTextStorageAccessTests(ITestOutputHelper output) : TempDatabaseTest { [Theory] @@ -146,9 +147,56 @@ public void Ace_picks_the_storage_form_from_the_uncompressed_length(int characte Assert.Equal(flag, raw[3] & 0xC0); } + // A memo steps two bytes at a time, so the theory above can only ever bracket the single-page edge: + // 3816 stays, 3818 chains, and 3817 was never asked. An OLE column takes any byte length, which closes + // it -- and the same instrument pins the inline edge exactly. Both rows live in ACE's own table, so the + // definition is out of the picture and the two engines' choices can be compared directly. Swept whole + // (1..100 and 3700..3899, every size): these four are where the answer changes. + [Theory] + [InlineData(64, 0x80)] // the last inline value + [InlineData(65, 0x40)] // the first to take a page + [InlineData(3816, 0x40)] // the last that stays on one page + [InlineData(3817, 0x00)] // the first chained -- the byte the memo theory cannot reach + public void Ace_and_libred_pick_the_same_storage_form_at_the_exact_byte_edges(int bytes, int flag) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "ole-edge-"); + using (OleDbConnection connection = AceTestDatabase.Open(path)) + { + using (OleDbCommand ddl = connection.CreateCommand()) + { + ddl.CommandText = "CREATE TABLE OleEdge (Id LONG, O OLEOBJECT)"; + ddl.ExecuteNonQuery(); + } + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = "INSERT INTO OleEdge (Id, O) VALUES (1, ?)"; + insert.Parameters.Add("o", OleDbType.LongVarBinary, bytes).Value = new byte[bytes]; + insert.ExecuteNonQuery(); + } + + using (var database = JetDatabase.Open(path, readOnly: false)) + database.OpenTable("OleEdge").Insert([2, new byte[bytes]]); + + using var read = JetDatabase.Open(path); + var definition = read.Catalog.FindTable("OleEdge")!; + int columnId = definition.Columns.Single(c => c.Name == "O").ColumnId; + using var channel = PageChannel.Open(path, readOnly: true); + List descriptors = RawDescriptors(channel, definition, columnId); + + Assert.Equal(2, descriptors.Count); + output.WriteLine($"{bytes} bytes: ACE 0x{descriptors[0][3] & 0xC0:X2}, LibRed 0x{descriptors[1][3] & 0xC0:X2}"); + Assert.Equal(flag, descriptors[0][3] & 0xC0); // ACE wrote first + Assert.Equal(flag, descriptors[1][3] & 0xC0); + } + /// The raw long-value descriptor ACE wrote into the row for one column. - private static byte[] RawDescriptor(PageChannel channel, Catalog.TableDef definition, int columnId) + private static byte[] RawDescriptor(PageChannel channel, Catalog.TableDef definition, int columnId) => + RawDescriptors(channel, definition, columnId).FirstOrDefault() + ?? throw new InvalidOperationException($"No long-value descriptor found for column id {columnId}."); + + /// Every row's long-value descriptor for one column, in row order. + private static List RawDescriptors(PageChannel channel, Catalog.TableDef definition, int columnId) { + var found = new List(); var decoder = new RowDecoder(definition.Columns, channel.Format); foreach (int number in new UsageMap(channel, definition).DataPages()) { @@ -159,9 +207,9 @@ private static byte[] RawDescriptor(PageChannel channel, Catalog.TableDef defini if (page.Rows[row].IsDeleted) continue; foreach (var descriptor in decoder.LongValueRaw(page.GetRow(row))) if (descriptor.Key == columnId) - return descriptor.Value[..12]; + found.Add(descriptor.Value[..12]); } } - throw new InvalidOperationException($"No long-value descriptor found for column id {columnId}."); + return found; } } diff --git a/test/LibRed.Core.AccessTests/LongValueLengthAccessTests.cs b/test/LibRed.Core.AccessTests/LongValueLengthAccessTests.cs index a91c28021..629d9bd84 100644 --- a/test/LibRed.Core.AccessTests/LongValueLengthAccessTests.cs +++ b/test/LibRed.Core.AccessTests/LongValueLengthAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; // ACE authors the database here; LibRed only reads. That direction is the point — it fails on the old // 24-bit reader and passes on the fixed one. Separately measured against ACE: 0x3FFFFFFF bytes are // accepted and 0x40000000 rejected, fixing the ceiling at 1 GiB — see long-values.md. +[Collection(AceCollection.Name)] public class LongValueLengthAccessTests : TempDatabaseTest { [Fact] diff --git a/test/LibRed.Core.AccessTests/LongValuePackingAccessTests.cs b/test/LibRed.Core.AccessTests/LongValuePackingAccessTests.cs index c2a239e1f..e1fc95c16 100644 --- a/test/LibRed.Core.AccessTests/LongValuePackingAccessTests.cs +++ b/test/LibRed.Core.AccessTests/LongValuePackingAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; /// values, that a full page leaves the free map while the current one stays free, and that both LibRed /// and Access read every value back. /// +[Collection(AceCollection.Name)] public class LongValuePackingAccessTests { private const int N = 20; // each ~310 bytes → LVAL, but ~10 fit per page diff --git a/test/LibRed.Core.AccessTests/MemoKeyEncodingTests.cs b/test/LibRed.Core.AccessTests/MemoKeyEncodingTests.cs index e4eadb633..d25ae9020 100644 --- a/test/LibRed.Core.AccessTests/MemoKeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/MemoKeyEncodingTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; // Byte-faithful: a Memo (Long Text) column IS indexable in Access, and its index key is the ordinary text // collation key over only the value's first 255 characters. Verified against keys Access itself wrote, // ascending and descending, including truncation and an "ignorable" character (apostrophe). +[Collection(AceCollection.Name)] public class MemoKeyEncodingTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/MinimumRecordSizeAccessTests.cs b/test/LibRed.Core.AccessTests/MinimumRecordSizeAccessTests.cs new file mode 100644 index 000000000..57ef32dec --- /dev/null +++ b/test/LibRed.Core.AccessTests/MinimumRecordSizeAccessTests.cs @@ -0,0 +1,135 @@ +using System.Data.OleDb; +using LibRed; +using LibRed.Catalog; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +// The other end of RecordSizeAccessTests. ACE pads an all-fixed row's fixed region out to two bytes, so the +// shortest record it will write is five: the 2-byte column count, that 2-byte region, and a 1-byte null +// bitmap. It is a floor and not an alignment - a three-BYTE table keeps its odd 3-byte region - and a row +// with a variable trailer is exempt, keeping a region of 0 or 1. +// +// LibRed used to size the region from the table definition alone, one or two bytes short. That is not +// cosmetic: an all-Boolean table of eight columns or fewer then encodes to a 3-byte record, and ACE reads +// every Boolean in it as False. The cliff is at four bytes rather than five - sixteen Booleans, whose bitmap +// is two bytes wide, read back correctly - but ACE's own writer never emits a record under five, so the short +// form is simply a shape its reader has never met, and it misreads it silently rather than refusing it. +[Collection(AceCollection.Name)] +public class MinimumRecordSizeAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + /// The record bytes of the single row in . + private static byte[] OnlyRecord(JetDatabase database, string table) + { + Table t = database.OpenTable(table); + int page = t.UsageMap.DataPages().Single(); + return database.ReadDataPage(page).GetRow(0).ToArray(); + } + + // Natural length is 2 (column count) + fixed bytes + ceil(columns / 8) (null bitmap). Booleans occupy no + // fixed bytes at all, which is how a shape gets a 0-byte region; nine of them is the one case that tells + // "pad the region to 2" apart from "pad the record to 5", because ACE writes it as six bytes. + [Theory] + [InlineData("YESNO", "True", 5)] // region 0 -> 2 + [InlineData("BYTE", "1", 5)] // region 1 -> 2 + [InlineData("BYTE, B YESNO", "1, True", 5)] // region 1 -> 2 + [InlineData("BYTE, B BYTE", "1, 2", 5)] // region 2, untouched + [InlineData("BYTE, B BYTE, C BYTE", "1, 2, 3", 6)] // region 3, odd and untouched + [InlineData("SHORT", "1", 5)] // region 2, untouched + [InlineData("YESNO, B YESNO, C YESNO, D YESNO, E YESNO, F YESNO, G YESNO, H YESNO, I YESNO", + "True, True, True, True, True, True, True, True, True", 6)] + [InlineData("TEXT(10)", "'ab'", 13)] // variable trailer: region 0, untouched + [InlineData("BYTE, B TEXT(10)", "1, 'ab'", 14)] // variable trailer: region 1, untouched + public void Ace_pads_an_all_fixed_record_to_five_bytes(string declaration, string values, int expected) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "minrec-ace-"); + string names = string.Join(", ", Enumerable.Range(0, values.Split(',').Length).Select(i => (char)('A' + i))); + + using (OleDbConnection connection = AceTestDatabase.Open(path)) + { + using (OleDbCommand ddl = connection.CreateCommand()) + { + ddl.CommandText = $"CREATE TABLE Narrow (A {declaration})"; + ddl.ExecuteNonQuery(); + } + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = $"INSERT INTO Narrow ({names}) VALUES ({values})"; + insert.ExecuteNonQuery(); + } + + using var database = JetDatabase.Open(path, readOnly: true); + byte[] record = OnlyRecord(database, "Narrow"); + output.WriteLine($"{declaration}: {record.Length} bytes, {Convert.ToHexString(record)}"); + Assert.Equal(expected, record.Length); + } + + // The pad is what ACE writes, so LibRed must write it too - byte for byte, not merely to the same length. + // Both tables are built and filled by each engine in turn in the same file, which also pins that the + // padding is a property of the record and not of the definition: the TDEF keeps the true fixed-row length + // (ACE stores 1 for a BYTE table while writing 5-byte rows into it), so nothing in the definition would + // carry this. + [Theory] + [InlineData(JetDataType.Boolean, "YESNO", true)] + [InlineData(JetDataType.Byte, "BYTE", (byte)7)] + public void Libred_writes_the_record_bytes_ace_writes(JetDataType type, string declaration, object value) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "minrec-parity-"); + + using (OleDbConnection connection = AceTestDatabase.Open(path)) + using (OleDbCommand ddl = connection.CreateCommand()) + { + ddl.CommandText = $"CREATE TABLE ByAce (V {declaration})"; + ddl.ExecuteNonQuery(); + ddl.CommandText = $"INSERT INTO ByAce (V) VALUES ({(value is bool ? "True" : value)})"; + ddl.ExecuteNonQuery(); + } + + using (var database = JetDatabase.Open(path, readOnly: false)) + { + database.CreateTable("ByLibRed", [new ColumnSpec("V", type, 1, IsFixedLength: true)]); + database.OpenTable("ByLibRed").Insert([value]); + } + + using var read = JetDatabase.Open(path, readOnly: true); + byte[] ace = OnlyRecord(read, "ByAce"), ours = OnlyRecord(read, "ByLibRed"); + output.WriteLine($"{declaration}: ACE {Convert.ToHexString(ace)}, LibRed {Convert.ToHexString(ours)}"); + Assert.Equal(Convert.ToHexString(ace), Convert.ToHexString(ours)); + } + + // The regression itself, and the only shape that was actually corrupt rather than merely short: a table + // of nothing but Booleans, whose row carries no fixed data at all. Without the pad ACE reports every one + // of them False. Eight columns and sixteen bracket the point where the null bitmap grows to two bytes and + // the record reaches four on its own. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(8)] + [InlineData(16)] + public void Ace_reads_a_boolean_only_table_libred_wrote(int columns) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, $"minrec-bits{columns}-"); + bool[] pattern = Enumerable.Range(0, columns).Select(i => i % 3 != 1).ToArray(); + + using (var database = JetDatabase.Open(path, readOnly: false)) + { + database.CreateTable("Bits", Enumerable.Range(0, columns) + .Select(i => new ColumnSpec($"F{i}", JetDataType.Boolean, 1, IsFixedLength: true)).ToList()); + database.OpenTable("Bits").Insert(pattern.Select(b => (object?)b).ToArray()); + } + + using (var read = JetDatabase.Open(path, readOnly: true)) + output.WriteLine($"{columns} Booleans: {Convert.ToHexString(OnlyRecord(read, "Bits"))}"); + + using OleDbConnection connection = AceTestDatabase.Open(path); + using OleDbCommand select = connection.CreateCommand(); + select.CommandText = "SELECT " + string.Join(", ", Enumerable.Range(0, columns).Select(i => $"F{i}")) + + " FROM Bits"; + using OleDbDataReader rows = select.ExecuteReader(); + + Assert.True(rows.Read()); + for (int i = 0; i < columns; i++) + Assert.Equal(pattern[i], rows.GetBoolean(i)); + Assert.False(rows.Read()); + } +} diff --git a/test/LibRed.Core.AccessTests/MixedCompressionAccessTests.cs b/test/LibRed.Core.AccessTests/MixedCompressionAccessTests.cs index 2d71701bc..23206c3f6 100644 --- a/test/LibRed.Core.AccessTests/MixedCompressionAccessTests.cs +++ b/test/LibRed.Core.AccessTests/MixedCompressionAccessTests.cs @@ -24,6 +24,7 @@ namespace LibRed.Core.Tests; // Scope: whatever ACE is installed. The provider is logged, since the scheme dates from Jet 4.0 and an // older engine may differ; 12.0 and 16.0 were checked by hand and agree byte for byte, but only one ACE is // installed at a time so the test cannot assert both. +[Collection(AceCollection.Name)] public class MixedCompressionAccessTests(ITestOutputHelper output) : TempDatabaseTest { // name, payload, stored bytes, mode switches in the payload diff --git a/test/LibRed.Core.AccessTests/MultiPageTdefParityAccessTests.cs b/test/LibRed.Core.AccessTests/MultiPageTdefParityAccessTests.cs index d1685ca0e..8644ca2f4 100644 --- a/test/LibRed.Core.AccessTests/MultiPageTdefParityAccessTests.cs +++ b/test/LibRed.Core.AccessTests/MultiPageTdefParityAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // // Stitching alone would not catch a wrong split — the same content can be divided differently and still // reassemble — so the definition page's own next-page link and free space are compared as well. +[Collection(AceCollection.Name)] public class MultiPageTdefParityAccessTests : TempDatabaseTest { [Theory] diff --git a/test/LibRed.Core.AccessTests/OrderByProcedureAccessTests.cs b/test/LibRed.Core.AccessTests/OrderByProcedureAccessTests.cs index 5facc1d08..e34c8c067 100644 --- a/test/LibRed.Core.AccessTests/OrderByProcedureAccessTests.cs +++ b/test/LibRed.Core.AccessTests/OrderByProcedureAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// Attribute=3 flag row (Flag bit 0x10, Name1 = the count) and each ORDER BY key as an /// Attribute=0x0B row (Expression = the column, Name1 = "d" for descending). Access runs it. /// +[Collection(AceCollection.Name)] public class OrderByProcedureAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/OwnedMapRecycleAccessTests.cs b/test/LibRed.Core.AccessTests/OwnedMapRecycleAccessTests.cs new file mode 100644 index 000000000..018f20546 --- /dev/null +++ b/test/LibRed.Core.AccessTests/OwnedMapRecycleAccessTests.cs @@ -0,0 +1,110 @@ +using System.Data.OleDb; +using System.Text; +using LibRed; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// Rebuilding an index (an ALTER COLUMN on an indexed column) gives it a new owned-pages usage-map +/// row, in the two writes described in docs/format/page-05-usage-maps.md §9: a fresh row appended and +/// stamped with the new root's bit, whose bytes are then abandoned, and a re-lay that reclaims the old +/// record. This compares the whole file against ACE's own answer, which is the only measurement that sees +/// both halves. +/// +/// +/// Each half was got wrong once, in a way weaker measurements could not catch. The abandoned copy decides +/// the diff on a single byte and lies inside the region free space already covers, so free-space accounting +/// says nothing about it. The re-lay only shows up when the recycled row is not the last — which an +/// ACE-built schema never produces, because a long-value column declared in CREATE TABLE takes its map +/// rows before the index's. Adding the Memo column afterwards is what puts the index's row in the middle, so +/// the last shape below is the one that matters and the first three are the shapes that were already covered. +/// +[Collection(AceCollection.Name)] +public class OwnedMapRecycleAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + private const int PageSize = 4096; + + [Theory] + [InlineData("one index", "CREATE TABLE T ( A LONG, B LONG );CREATE INDEX ixB ON T (B)", + "INSERT INTO T (A,B) VALUES (11,22)", "")] + [InlineData("two indexes on the target", "CREATE TABLE T ( A LONG, B LONG );CREATE INDEX ix1 ON T (B);CREATE INDEX ix2 ON T (B)", + "INSERT INTO T (A,B) VALUES (11,22)", "")] + [InlineData("memo declared with the table", "CREATE TABLE T ( A LONG, M MEMO, B LONG );CREATE INDEX ixB ON T (B)", + "INSERT INTO T (A,M,B) VALUES (11,'hi',22)", "")] + // The index's map row is no longer last: its record is reclaimed and the long-value maps slide up. + [InlineData("memo added after the index", "CREATE TABLE T ( A LONG, B LONG );CREATE INDEX ixB ON T (B)", + "INSERT INTO T (A,B) VALUES (11,22)", "ALTER TABLE T ADD COLUMN M LONGTEXT")] + public void Libred_rebuilds_the_index_byte_for_byte_with_ace(string label, string create, string insert, string then) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "recycle-start-"); + string ace = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "recycle-ace-"); + string libred = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "recycle-lib-"); + try + { + // ACE builds the schema, so both engines start from a file only ACE has written. + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + foreach (string statement in create.Split(';')) Exec(connection, statement); + Exec(connection, insert); + if (then.Length > 0) Exec(connection, then); + } + + File.Copy(start, ace, overwrite: true); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, "ALTER TABLE T ALTER COLUMN B DOUBLE"); + + File.Copy(start, libred, overwrite: true); + using (var database = JetDatabase.Open(libred, readOnly: false)) + database.AlterColumn("T", "B", new ColumnSpec("B", JetDataType.Double, 8, IsFixedLength: true)); + + output.WriteLine($"{label}: comparing {new FileInfo(ace).Length / PageSize} pages"); + Assert.Equal("", Difference(ace, libred)); + } + finally + { + TemporaryDatabase.Delete(start); + TemporaryDatabase.Delete(ace); + TemporaryDatabase.Delete(libred); + } + } + + /// Every differing byte of every page, except the two environmental spots: page 0 carries the + /// database modification counter, and MSysObjects' data page (owner 2) carries the table's DateUpdate + /// wall clock. What remains covers the TDEF, the data pages, the index B-tree, the usage maps and the + /// global free map. + private static string Difference(string acePath, string libredPath) + { + byte[] ace = File.ReadAllBytes(acePath), libred = File.ReadAllBytes(libredPath); + var differences = new StringBuilder(); + int pages = Math.Max(ace.Length, libred.Length) / PageSize; + for (int page = 1; page < pages; page++) + { + int at = page * PageSize; + bool inAce = at + PageSize <= ace.Length, inLibRed = at + PageSize <= libred.Length; + if (inAce && inLibRed && BitConverter.ToInt32(ace, at + 4) == 2) continue; + if (!inAce || !inLibRed) + { + differences.AppendLine($"page {page}: present in {(inAce ? "ACE" : "LibRed")} only"); + continue; + } + for (int i = 0, shown = 0; i < PageSize && shown < 8; i++) + if (ace[at + i] != libred[at + i]) + { + differences.AppendLine( + $"page {page} (type 0x{ace[at]:X2} owner {BitConverter.ToInt32(ace, at + 4)}) " + + $"+0x{i:X3}: ace={ace[at + i]:X2} libred={libred[at + i]:X2}"); + shown++; + } + } + return differences.ToString(); + } + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/PackedLongValueReleaseAccessTests.cs b/test/LibRed.Core.AccessTests/PackedLongValueReleaseAccessTests.cs new file mode 100644 index 000000000..88adfa2b0 --- /dev/null +++ b/test/LibRed.Core.AccessTests/PackedLongValueReleaseAccessTests.cs @@ -0,0 +1,136 @@ +using System.Data.OleDb; +using System.Text; +using LibRed; +using LibRed.IO; +using LibRed.Pages; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// Several small long values (the single-page form, up to 3,816 bytes) pack onto one LVAL page. Deleting a +/// row retires its value's row there to a 0-length deleted+overflow tombstone and re-lays the page, and once +/// the last value on it is gone the page is stamped and freed. +/// That is where the 0x09 pages in real Access files come from — see +/// docs/format/page-05-usage-maps.md §9. +/// +/// +/// A chained value owns its pages outright and gives them back at 0x01, which is why no +/// experiment using a memo large enough to chain ever produced a 0x09, and why the type went +/// unexplained for so long. Both bands are covered below. +/// +[Collection(AceCollection.Name)] +public class PackedLongValueReleaseAccessTests(ITestOutputHelper output) : TempDatabaseTest +{ + private const int PageSize = 4096, Packed = 400, Chained = 20_000; + + [Theory] + [InlineData(Packed, 12, 4)] // some rows: the shared pages survive, compacted + [InlineData(Packed, 12, 12)] // every row: the pages empty out and are released + [InlineData(Packed, 40, 40)] // several pages' worth + [InlineData(Chained, 6, 6)] // chained values: pages go back at 0x01, never 0x09 + public void Libred_releases_packed_long_values_byte_for_byte_with_ace(int chars, int rows, int remove) + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "lval-start-"); + string ace = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "lval-ace-"); + string libred = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "lval-lib-"); + try + { + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + Exec(connection, "CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, M MEMO)"); + for (int i = 1; i <= rows; i++) + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = "INSERT INTO T (Id, M) VALUES (?, ?)"; + insert.Parameters.Add("id", OleDbType.Integer).Value = i; + insert.Parameters.Add("m", OleDbType.LongVarWChar, chars).Value = new string((char)('a' + i % 26), chars); + insert.ExecuteNonQuery(); + } + } + + File.Copy(start, ace, overwrite: true); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, $"DELETE FROM T WHERE Id <= {remove}"); + + File.Copy(start, libred, overwrite: true); + using (var database = JetDatabase.Open(libred, readOnly: false)) + { + Table table = database.OpenTable("T"); + int id = table.Definition.FindColumn("Id")!.Index; + foreach ((RowId rowId, object?[] values) in table.Rows().WithIds().ToList()) + if (Convert.ToInt32(values[id]) <= remove) + table.Delete(rowId); + } + + output.WriteLine($"ACE {Census(ace)}"); + output.WriteLine($"LibRed {Census(libred)}"); + Assert.Equal(Census(ace), Census(libred)); + Assert.Equal("", Difference(ace, libred)); + } + finally + { + TemporaryDatabase.Delete(start); + TemporaryDatabase.Delete(ace); + TemporaryDatabase.Delete(libred); + } + } + + /// How many long-value pages are live and how many were released, so a shape difference is + /// reported as such rather than as a wall of bytes. + private static string Census(string path) + { + using var channel = PageChannel.Open(path); + var buffer = new byte[channel.PageSize]; + int live = 0, released = 0; + for (int page = 0; page < channel.PageCount; page++) + { + channel.ReadPage(page, buffer); + if (BitConverter.ToUInt32(buffer, channel.Format.DataOwnerOffset) != 0x4C41564C) continue; + if (buffer[0] == (byte)PageType.DataPage) live++; + else if (buffer[0] == (byte)PageType.ReleasedLongValuePage) released++; + } + return $"lval live={live} released={released}"; + } + + /// Every differing byte of every page, skipping page 0 (the modification counter), MSysObjects' + /// data page (the table's DateUpdate wall clock) and index pages — removing entries leaves the two + /// engines with identical index content and byte-different pages, which is recorded and accepted in + /// page-03-04 §10.4a. + private static string Difference(string acePath, string libredPath) + { + byte[] ace = File.ReadAllBytes(acePath), libred = File.ReadAllBytes(libredPath); + var differences = new StringBuilder(); + int pages = Math.Max(ace.Length, libred.Length) / PageSize; + for (int page = 1; page < pages; page++) + { + int at = page * PageSize; + bool inAce = at + PageSize <= ace.Length, inLibRed = at + PageSize <= libred.Length; + if (!inAce || !inLibRed) + { + differences.AppendLine($"page {page}: present in {(inAce ? "ACE" : "LibRed")} only"); + continue; + } + if (BitConverter.ToInt32(ace, at + 4) == 2) continue; + if (ace[at] is (byte)PageType.IntermediateIndexPage or (byte)PageType.LeafIndexPage) continue; + + for (int i = 0, shown = 0; i < PageSize && shown < 8; i++) + if (ace[at + i] != libred[at + i]) + { + differences.AppendLine( + $"page {page} (type 0x{ace[at]:X2} owner {BitConverter.ToInt32(ace, at + 4)}) " + + $"+0x{i:X3}: ace={ace[at + i]:X2} libred={libred[at + i]:X2}"); + shown++; + } + } + return differences.ToString(); + } + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/PrimaryKeyNameAccessTests.cs b/test/LibRed.Core.AccessTests/PrimaryKeyNameAccessTests.cs index 54fe367b1..17bafb389 100644 --- a/test/LibRed.Core.AccessTests/PrimaryKeyNameAccessTests.cs +++ b/test/LibRed.Core.AccessTests/PrimaryKeyNameAccessTests.cs @@ -13,6 +13,7 @@ namespace LibRed.Core.Tests; /// EF Core scaffolder reads). When unnamed, LibRed writes its stable "PrimaryKey" fallback, which Access reads /// back unchanged (ACE creating an unnamed PK via SQL would instead generate a random "Index_"). /// +[Collection(AceCollection.Name)] public class PrimaryKeyNameAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ProcedureParameterAccessTests.cs b/test/LibRed.Core.AccessTests/ProcedureParameterAccessTests.cs index faa1739c0..39ab130ed 100644 --- a/test/LibRed.Core.AccessTests/ProcedureParameterAccessTests.cs +++ b/test/LibRed.Core.AccessTests/ProcedureParameterAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// declared parameter as an Attribute=2 MSysQueries row (Name1 = name, Flag = Jet type code, e.g. /// 8 = DateTime). Access runs it and honours the supplied parameter values. /// +[Collection(AceCollection.Name)] public class ProcedureParameterAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/PropertyNamePoolOrderTests.cs b/test/LibRed.Core.AccessTests/PropertyNamePoolOrderTests.cs index 6e256885b..f5c4152f5 100644 --- a/test/LibRed.Core.AccessTests/PropertyNamePoolOrderTests.cs +++ b/test/LibRed.Core.AccessTests/PropertyNamePoolOrderTests.cs @@ -11,6 +11,7 @@ namespace LibRed.Core.Tests; // Proven against ACE with three names whose first-appearance order (Required, DefaultValue, CheckConstraints) // is deliberately not alphabetical (which would be CheckConstraints, DefaultValue, Required). Guards against a // future "tidy-up" that sorts the pool in PropertyBlob.Write (which uses Distinct() = first appearance). +[Collection(AceCollection.Name)] public class PropertyNamePoolOrderTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/QualifiedStarViewAccessTests.cs b/test/LibRed.Core.AccessTests/QualifiedStarViewAccessTests.cs index 15ab9a4ca..937d2c8f5 100644 --- a/test/LibRed.Core.AccessTests/QualifiedStarViewAccessTests.cs +++ b/test/LibRed.Core.AccessTests/QualifiedStarViewAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// Access runs a LibRed-created view whose projection uses a table-qualified star (Products.*), /// stored as the column's verbatim Expression — the shape of Northwind's "Alphabetical list of products". /// +[Collection(AceCollection.Name)] public class QualifiedStarViewAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/RandomAutoNumberAccessTests.cs b/test/LibRed.Core.AccessTests/RandomAutoNumberAccessTests.cs index 2df873775..92b5ae2a0 100644 --- a/test/LibRed.Core.AccessTests/RandomAutoNumberAccessTests.cs +++ b/test/LibRed.Core.AccessTests/RandomAutoNumberAccessTests.cs @@ -8,6 +8,7 @@ namespace LibRed.Core.Tests; // Byte-faithful check: a LibRed-written "Random" AutoNumber (COUNTER DEFAULT GenUniqueID()) is read by Access. // ACE opens the file without repair, sees it as a proper AutoNumber (rejects a supplied Id, auto-assigns on // insert), and continues issuing random-looking (non-sequential) ids of its own. +[Collection(AceCollection.Name)] public class RandomAutoNumberAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/RecordSizeAccessTests.cs b/test/LibRed.Core.AccessTests/RecordSizeAccessTests.cs index 2a807dcd3..f349f576d 100644 --- a/test/LibRed.Core.AccessTests/RecordSizeAccessTests.cs +++ b/test/LibRed.Core.AccessTests/RecordSizeAccessTests.cs @@ -16,6 +16,7 @@ namespace LibRed.Core.Tests; // throwing ArgumentOutOfRangeException out of the offset arithmetic. // // The 20-byte reserve between 4060 and what the page holds is measured, not explained. +[Collection(AceCollection.Name)] public class RecordSizeAccessTests(ITestOutputHelper output) : TempDatabaseTest { /// Row overhead for a table of TEXT columns plus a LONG key: the diff --git a/test/LibRed.Core.AccessTests/RefActionAccessTests.cs b/test/LibRed.Core.AccessTests/RefActionAccessTests.cs index 47e4dfaf5..a0954dce9 100644 --- a/test/LibRed.Core.AccessTests/RefActionAccessTests.cs +++ b/test/LibRed.Core.AccessTests/RefActionAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// A LibRed-written ON DELETE SET NULL relationship (grbit 0x2000) is byte-faithful: Access reads it and /// applies the SET NULL itself when it deletes the parent. /// +[Collection(AceCollection.Name)] public class RefActionAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ReleaseAtCloseAccessTests.cs b/test/LibRed.Core.AccessTests/ReleaseAtCloseAccessTests.cs new file mode 100644 index 000000000..459192b86 --- /dev/null +++ b/test/LibRed.Core.AccessTests/ReleaseAtCloseAccessTests.cs @@ -0,0 +1,88 @@ +using System.Buffers.Binary; +using System.Data.OleDb; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +// A session's freed pages go back to the global free map at close, and the close lengthens the released-pages map +// to cover the highest page released (docs/format/page-05-usage-maps.md §9.1). The same DROP TABLE through ACE and +// through LibRed, each on its own copy, must leave page 1 — both global maps — byte for byte the same. +[Collection(AceCollection.Name)] +public class ReleaseAtCloseAccessTests : TempDatabaseTest +{ + [Fact] + public void A_dropped_table_leaves_both_global_maps_as_ace_leaves_them() + { + string start = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-start-"); + using (OleDbConnection connection = AceTestDatabase.Open(start)) + { + Exec(connection, "CREATE TABLE Doomed (Id LONG CONSTRAINT pk PRIMARY KEY, M MEMO)"); + for (int i = 1; i <= 40; i++) + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = "INSERT INTO Doomed (Id, M) VALUES (?, ?)"; + insert.Parameters.Add("id", OleDbType.Integer).Value = i; + insert.Parameters.Add("m", OleDbType.LongVarWChar, 20_000).Value = new string((char)('a' + i % 26), 20_000); + insert.ExecuteNonQuery(); + } + } + byte[] before = ReadPage(start, 1); + + string ace = TemporaryDatabase.CopyPath(start, "release-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + Exec(connection, "DROP TABLE Doomed"); + + string libred = TemporaryDatabase.CopyPath(start, "release-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + Assert.True(db.DropTable("Doomed")); + + byte[] acePage = ReadPage(ace, 1), libredPage = ReadPage(libred, 1); + Assert.True(ReleasedLength(acePage) > ReleasedLength(before), "the drop did not lengthen ACE's released map"); + Assert.Equal(ReleasedLength(acePage), ReleasedLength(libredPage)); + Assert.True(acePage.AsSpan().SequenceEqual(libredPage), FreeMapDifference(start, before, acePage, libredPage)); + } + + /// The pages whose free bit differs between the two engines, with each page's type, owner and free + /// bit in the file before the drop. + private static string FreeMapDifference(string start, byte[] before, byte[] acePage, byte[] libredPage) + { + int beforeOffset = BinaryPrimitives.ReadUInt16LittleEndian(before.AsSpan(14)) & 0x1FFF; + int offset = BinaryPrimitives.ReadUInt16LittleEndian(acePage.AsSpan(14)) & 0x1FFF; + int first = BinaryPrimitives.ReadInt32LittleEndian(acePage.AsSpan(offset + 1)); + var lines = new List(); + for (int i = offset + 5; i < acePage.Length; i++) + for (int bit = 0; bit < 8; bit++) + if (((acePage[i] ^ libredPage[i]) & (1 << bit)) != 0) + { + int page = first + (i - offset - 5) * 8 + bit; + byte[] bytes = ReadPage(start, page); + int beforeByte = beforeOffset + 5 + (page - first) / 8; + bool freeBefore = (before[beforeByte] & (1 << ((page - first) % 8))) != 0; + lines.Add($"page {page} (type 0x{bytes[0]:X2} owner {BitConverter.ToInt32(bytes, 4)}, " + + $"{(freeBefore ? "free" : "used")} before the drop): " + + $"free in {((acePage[i] & (1 << bit)) != 0 ? "ACE" : "LibRed")} only"); + } + return lines.Count == 0 ? "page 1 differs outside the free map" : string.Join("; ", lines); + } + + private static int ReleasedLength(byte[] page1) => + (BinaryPrimitives.ReadUInt16LittleEndian(page1.AsSpan(14)) & 0x1FFF) + - (BinaryPrimitives.ReadUInt16LittleEndian(page1.AsSpan(16)) & 0x1FFF); + + private static byte[] ReadPage(string path, int page) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + var bytes = new byte[4096]; + s.Position = page * 4096L; + s.ReadExactly(bytes); + return bytes; + } + + private static void Exec(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Core.AccessTests/RenameNameValidationAccessTests.cs b/test/LibRed.Core.AccessTests/RenameNameValidationAccessTests.cs index 3c9fbfc65..698181b0b 100644 --- a/test/LibRed.Core.AccessTests/RenameNameValidationAccessTests.cs +++ b/test/LibRed.Core.AccessTests/RenameNameValidationAccessTests.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // "Unrecognized database format". A 100-character table name happened to survive, and the bracketed names // only broke SQL that tried to reference them, so the column case is the one that did real damage - but the // create path refuses all of them and a rename has no reason to be more permissive. +[Collection(AceCollection.Name)] public class RenameNameValidationAccessTests : TempDatabaseTest { private static List Specs() => diff --git a/test/LibRed.Core.AccessTests/RequiredColumnTests.cs b/test/LibRed.Core.AccessTests/RequiredColumnTests.cs index 00dd95501..09a6866f6 100644 --- a/test/LibRed.Core.AccessTests/RequiredColumnTests.cs +++ b/test/LibRed.Core.AccessTests/RequiredColumnTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// blob (absent for a nullable column). LibRed writes it byte-faithfully, reads it back onto /// , and ACE enforces it on a LibRed-created table. /// +[Collection(AceCollection.Name)] public class RequiredColumnTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ReverseDiacriticProbeTest.cs b/test/LibRed.Core.AccessTests/ReverseDiacriticProbeTest.cs index b34f9ba49..26fbb539e 100644 --- a/test/LibRed.Core.AccessTests/ReverseDiacriticProbeTest.cs +++ b/test/LibRed.Core.AccessTests/ReverseDiacriticProbeTest.cs @@ -15,6 +15,7 @@ namespace LibRed.Core.Tests; // // If ACE agrees, French costs one flag rather than a table of overrides, and so do the other orders that // carry it. Opt-in via LIBRED_REVERSE_DW=1. +[Collection(AceCollection.Name)] public class ReverseDiacriticProbeTest(ITestOutputHelper output) { // The measured French keys, as ACE stored them in an index. French tailors no letter at all: it is @@ -230,6 +231,7 @@ static int Compare(byte[] a, byte[] b) /// yet encode that order's keys. private static bool CreateWithDao(string path, string langId) { + AceTestDatabase.ReleaseAbandonedComObjects(); object? engine = null; foreach (int n in (int[])[120, 36]) { diff --git a/test/LibRed.Core.AccessTests/RowInserterTests.cs b/test/LibRed.Core.AccessTests/RowInserterTests.cs index 69968e713..2f6bf8fff 100644 --- a/test/LibRed.Core.AccessTests/RowInserterTests.cs +++ b/test/LibRed.Core.AccessTests/RowInserterTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class RowInserterTests { private static string CopyToTemp() diff --git a/test/LibRed.Core.AccessTests/SelfReferencingForeignKeyAccessTests.cs b/test/LibRed.Core.AccessTests/SelfReferencingForeignKeyAccessTests.cs index 9d1f70869..cd84bae87 100644 --- a/test/LibRed.Core.AccessTests/SelfReferencingForeignKeyAccessTests.cs +++ b/test/LibRed.Core.AccessTests/SelfReferencingForeignKeyAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// Employees.EmployeeID). Both relationship ends live in the one table's TDEF; the backing index over the /// FK column is back-filled from the existing rows. Access reads and enforces the relationship. /// +[Collection(AceCollection.Name)] public class SelfReferencingForeignKeyAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/SortKeyComparisonProbeTest.cs b/test/LibRed.Core.AccessTests/SortKeyComparisonProbeTest.cs index f95be7cfe..946fd0c38 100644 --- a/test/LibRed.Core.AccessTests/SortKeyComparisonProbeTest.cs +++ b/test/LibRed.Core.AccessTests/SortKeyComparisonProbeTest.cs @@ -22,6 +22,7 @@ namespace LibRed.Core.Tests; // platform unless System.Globalization.UseNls is set, so it is NOT the Win32 sort key. // 4. LCMapStringEx(LCMAP_SORTKEY) - the Win32 NLS API Jet itself used, called directly so the comparison // does not depend on which globalization backend .NET happens to be using. +[Collection(AceCollection.Name)] public class SortKeyComparisonProbeTest(ITestOutputHelper output) { private const uint LcmapSortkey = 0x00000400; diff --git a/test/LibRed.Core.AccessTests/SortKeyTableV0GeneratorTest.cs b/test/LibRed.Core.AccessTests/SortKeyTableV0GeneratorTest.cs index 74a5e8ace..61ea2f01c 100644 --- a/test/LibRed.Core.AccessTests/SortKeyTableV0GeneratorTest.cs +++ b/test/LibRed.Core.AccessTests/SortKeyTableV0GeneratorTest.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // hand-written tables in JetTextCollation and the compact per-block strings can hold, hence a resource. // // Opt-in via LIBRED_GENERATE_V0=1: it inserts ~63,000 rows through ACE and rewrites a checked-in binary. +[Collection(AceCollection.Name)] public class SortKeyTableV0GeneratorTest(ITestOutputHelper output) { private const string ResourcePath = "src/LibRed/LibRed.Core/Resources/SortKeyTableV0.bin"; diff --git a/test/LibRed.Core.AccessTests/SortKeyTableV1OverrideGeneratorTest.cs b/test/LibRed.Core.AccessTests/SortKeyTableV1OverrideGeneratorTest.cs index 8bb48da28..45e94269d 100644 --- a/test/LibRed.Core.AccessTests/SortKeyTableV1OverrideGeneratorTest.cs +++ b/test/LibRed.Core.AccessTests/SortKeyTableV1OverrideGeneratorTest.cs @@ -19,6 +19,7 @@ namespace LibRed.Core.Tests; // weight ACE implies wherever they differ. That is the same answer v0 needed, at 1% of the size. // // Opt-in via LIBRED_GENERATE_V1=1: it inserts ~63,000 rows through ACE and rewrites a checked-in binary. +[Collection(AceCollection.Name)] public class SortKeyTableV1OverrideGeneratorTest(ITestOutputHelper output) { private const string ResourcePath = "src/LibRed/LibRed.Core/Resources/SortKeyTableV1Overrides.bin"; diff --git a/test/LibRed.Core.AccessTests/SpanishCollationProbeTest.cs b/test/LibRed.Core.AccessTests/SpanishCollationProbeTest.cs index 8bf64ff7d..5533244d3 100644 --- a/test/LibRed.Core.AccessTests/SpanishCollationProbeTest.cs +++ b/test/LibRed.Core.AccessTests/SpanishCollationProbeTest.cs @@ -21,6 +21,7 @@ namespace LibRed.Core.Tests; // +2 almost everywhere, leaving 0x4E between C and D, 0x5F between L and M, and 0x63 between N and O. // Gaps are the norm rather than a Spanish reservation, so landing on exactly those three would be a // real result: it would make the compacted v0 table's gaps insertion slots for language letters. +[Collection(AceCollection.Name)] public class SpanishCollationProbeTest(ITestOutputHelper output) { // Single letters and digraphs read the weights directly; the words show the ordering they produce. diff --git a/test/LibRed.Core.AccessTests/StoredQueryKindAccessTests.cs b/test/LibRed.Core.AccessTests/StoredQueryKindAccessTests.cs index 07091184f..c31580b30 100644 --- a/test/LibRed.Core.AccessTests/StoredQueryKindAccessTests.cs +++ b/test/LibRed.Core.AccessTests/StoredQueryKindAccessTests.cs @@ -12,6 +12,7 @@ namespace LibRed.Core.Tests; /// real-world databases that was 221 of 592 stored queries. These author each shape through DAO, exactly as /// the Access UI does, and check LibRed classifies and rebuilds it. /// +[Collection(AceCollection.Name)] public class StoredQueryKindAccessTests { [Fact] @@ -163,13 +164,7 @@ private static int QueryObjectId(JetDatabase db, string queryName) /// skipped) where DAO is not registered. private static bool Author(string path, params (string Name, string Sql)[] queries) { - object? engine = null; - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { engine = Activator.CreateInstance(type); break; } catch (Exception) { } - } + object? engine = AceTestDatabase.CreateDaoEngine(); if (engine is null) return false; object database = Invoke(engine, "OpenDatabase", path, false, false, "")!; diff --git a/test/LibRed.Core.AccessTests/TailoringGeneratorProbeTest.cs b/test/LibRed.Core.AccessTests/TailoringGeneratorProbeTest.cs index 2ddaf1c5d..565c9f0ee 100644 --- a/test/LibRed.Core.AccessTests/TailoringGeneratorProbeTest.cs +++ b/test/LibRed.Core.AccessTests/TailoringGeneratorProbeTest.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // Generate_general_coverage — everything General v0 does not yet encode, per block // Generate_diacritic_weights_missing_from_general — combining marks whose secondary weight is missing // Generate_tailoring_for — one locale's overrides against General +[Collection(AceCollection.Name)] public class TailoringGeneratorProbeTest(ITestOutputHelper output) { /// The blocks worth sweeping for General v0 coverage: everything a Jet/ACE text column is diff --git a/test/LibRed.Core.AccessTests/TdefReserveSpillAccessTests.cs b/test/LibRed.Core.AccessTests/TdefReserveSpillAccessTests.cs new file mode 100644 index 000000000..4814b3c69 --- /dev/null +++ b/test/LibRed.Core.AccessTests/TdefReserveSpillAccessTests.cs @@ -0,0 +1,76 @@ +using System.Data.OleDb; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// A table definition's chain holds the definition and then its 8-byte trailing reserve, which spills onto a page of +/// its own when it does not fit — so a definition within 8 bytes of a page boundary has a continuation page holding +/// reserve bytes and no definition (docs/format/page-02a-tdef.md §3.2). LibRed reads such a chain from ACE, and lays +/// its own out the same way: the same pages, each with the same free space. +/// +[Collection(AceCollection.Name)] +public class TdefReserveSpillAccessTests : TempDatabaseTest +{ + // 115 LONG columns make a 4,086-byte definition; each character added to the first column's name adds two. + // 232 make an 8,181-byte one, whose reserve spills onto a third page. + [Theory] + [InlineData(115, 1, 4088, 1)] + [InlineData(115, 2, 4090, 2)] + [InlineData(115, 5, 4096, 2)] + [InlineData(115, 6, 4098, 2)] + [InlineData(232, 0, 8181, 3)] + public void A_definition_near_a_page_boundary_is_laid_out_as_ace_lays_it(int columns, int longerName, int length, int pages) + { + string ace = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "tdefspill-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(ace)) + using (OleDbCommand create = connection.CreateCommand()) + { + create.CommandText = $"CREATE TABLE L ({string.Join(", ", Names(columns, longerName).Select(n => n + " LONG"))})"; + create.ExecuteNonQuery(); + create.CommandText = "INSERT INTO L (Id) VALUES (7)"; + create.ExecuteNonQuery(); + } + + string libred = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "tdefspill-lib-"); + using (var db = JetDatabase.Open(libred, readOnly: false)) + { + db.CreateTable("L", Names(columns, longerName).Select(n => new ColumnSpec(n, JetDataType.Int32, 4, IsFixedLength: true)).ToArray()); + db.OpenTable("L").Insert([7, .. Enumerable.Repeat(null, columns - 1)]); + } + + (int Length, IReadOnlyList Free) aceChain = Chain(ace), libredChain = Chain(libred); + Assert.Equal(length, aceChain.Length); + Assert.Equal(pages, aceChain.Free.Count); + Assert.Equal(aceChain.Length, libredChain.Length); + Assert.Equal(aceChain.Free, libredChain.Free); + + // Each engine reads the other's table. + using (var db = JetDatabase.Open(ace)) + { + Assert.Equal(columns, db.Catalog.FindTable("L")!.Columns.Count); + Assert.Equal(7, db.OpenTable("L").Rows().Single()[0]); + } + using OleDbConnection check = AceTestDatabase.Open(libred); + using OleDbCommand read = check.CreateCommand(); + read.CommandText = "SELECT Id FROM L"; + Assert.Equal(7, read.ExecuteScalar()); + } + + private static IEnumerable Names(int columns, int longerName) => + new[] { "Id", "c001" + new string('x', longerName) }.Concat(Enumerable.Range(2, columns - 2).Select(i => $"c{i:D3}")); + + /// Table L's definition length and each page's free space, first page first — read from the raw file, + /// so LibRed's own chain reader is not the judge of it. + private static (int Length, IReadOnlyList Free) Chain(string path) + { + int first; + using (var db = JetDatabase.Open(path)) first = db.Catalog.FindTable("L")!.DefinitionPage; + byte[] file = File.ReadAllBytes(path); + var free = new List(); + for (int page = first, n = 0; page != 0 && n < 10; page = BitConverter.ToInt32(file, page * 4096 + 4), n++) + free.Add(BitConverter.ToUInt16(file, page * 4096 + 2)); + return (BitConverter.ToInt32(file, first * 4096 + 8), free); + } +} diff --git a/test/LibRed.Core.AccessTests/TextKeyEncodingTests.cs b/test/LibRed.Core.AccessTests/TextKeyEncodingTests.cs index e7a0c53a2..a6e4f20d2 100644 --- a/test/LibRed.Core.AccessTests/TextKeyEncodingTests.cs +++ b/test/LibRed.Core.AccessTests/TextKeyEncodingTests.cs @@ -6,6 +6,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class TextKeyEncodingTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/TextPrimaryKeyInsertTests.cs b/test/LibRed.Core.AccessTests/TextPrimaryKeyInsertTests.cs index e4274fef9..401f725e5 100644 --- a/test/LibRed.Core.AccessTests/TextPrimaryKeyInsertTests.cs +++ b/test/LibRed.Core.AccessTests/TextPrimaryKeyInsertTests.cs @@ -5,6 +5,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class TextPrimaryKeyInsertTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/ThaiReorderProbeTest.cs b/test/LibRed.Core.AccessTests/ThaiReorderProbeTest.cs index 2325db85c..cc380c046 100644 --- a/test/LibRed.Core.AccessTests/ThaiReorderProbeTest.cs +++ b/test/LibRed.Core.AccessTests/ThaiReorderProbeTest.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // is not evidence of a small tailoring. This measures Thai text rather than Thai characters. // // Opt-in via LIBRED_THAI=1. +[Collection(AceCollection.Name)] public class ThaiReorderProbeTest(ITestOutputHelper output) { [Fact] diff --git a/test/LibRed.Core.AccessTests/TransactionPhysicalRollbackAccessTests.cs b/test/LibRed.Core.AccessTests/TransactionPhysicalRollbackAccessTests.cs index 5b392a61f..1bcba7877 100644 --- a/test/LibRed.Core.AccessTests/TransactionPhysicalRollbackAccessTests.cs +++ b/test/LibRed.Core.AccessTests/TransactionPhysicalRollbackAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; /// Rollback coverage for page allocation, B-tree splits, relocation, and long-value ownership. +[Collection(AceCollection.Name)] public class TransactionPhysicalRollbackAccessTests { [Fact] diff --git a/test/LibRed.Core.AccessTests/TransactionSavepointAndEncryptionAccessTests.cs b/test/LibRed.Core.AccessTests/TransactionSavepointAndEncryptionAccessTests.cs index 08f49dec1..b0fea7b1a 100644 --- a/test/LibRed.Core.AccessTests/TransactionSavepointAndEncryptionAccessTests.cs +++ b/test/LibRed.Core.AccessTests/TransactionSavepointAndEncryptionAccessTests.cs @@ -7,6 +7,7 @@ namespace LibRed.Core.Tests; +[Collection(AceCollection.Name)] public class TransactionSavepointAndEncryptionAccessTests { [Theory] diff --git a/test/LibRed.Core.AccessTests/UpdateAccessTests.cs b/test/LibRed.Core.AccessTests/UpdateAccessTests.cs index 0306419d7..c7142c625 100644 --- a/test/LibRed.Core.AccessTests/UpdateAccessTests.cs +++ b/test/LibRed.Core.AccessTests/UpdateAccessTests.cs @@ -10,6 +10,7 @@ namespace LibRed.Core.Tests; /// LibRed rewrites a row in place (row id preserved) and Access reads the updated values back — including a /// memo that grew onto an LVAL page and a variable-text column that grew the row (page repacked in place). /// +[Collection(AceCollection.Name)] public class UpdateAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); @@ -164,6 +165,74 @@ public void Access_reads_a_memo_after_libred_reclaims_and_reuses_lval_pages() finally { TemporaryDatabase.Delete(path); } } + [Fact] + public void Access_scans_every_value_after_repeated_mixed_long_value_replacement() + { + const int rowCount = 18, columns = 3, replacements = 240; + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "mixed-lval-churn-"); + var expected = new Dictionary(); + + static string Value(int row, int column, int generation) + { + int length = (row * 17 + column * 31 + generation * 13) % 4 switch + { + 0 => 30, // inline + 1 => 350, // packed LVAL + 2 => 5_000, // chained LVAL + _ => 20_000, // multi-page chain + }; + return $"r{row}-c{column}-g{generation}:" + new string((char)('a' + (row + column + generation) % 26), length); + } + + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("LvalChurn", + [ + new("Id", JetDataType.Int32, 4, IsFixedLength: true), + new("M0", JetDataType.Memo, 0, IsFixedLength: false), + new("M1", JetDataType.Memo, 0, IsFixedLength: false), + new("M2", JetDataType.Memo, 0, IsFixedLength: false), + ], primaryKey: ["Id"]); + Table table = db.OpenTable("LvalChurn"); + for (int row = 1; row <= rowCount; row++) + { + string[] values = Enumerable.Range(0, columns).Select(column => Value(row, column, 0)).ToArray(); + expected.Add(row, values); + table.Insert([row, .. values]); + } + + var random = new Random(20260912); + for (int generation = 1; generation <= replacements; generation++) + { + int row = random.Next(1, rowCount + 1), column = random.Next(columns); + RowId id = table.Rows().WithIds() + .Single(item => Convert.ToInt32(item.Values[0]) == row).Id; + object?[] values = table.GetRow(id)!; + string replacement = Value(row, column, generation); + values[column + 1] = replacement; + expected[row][column] = replacement; + table.Update(id, values, new HashSet { column + 1 }); + } + } + + using var connection = OpenOleDb(path); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT Id, M0, M1, M2 FROM LvalChurn ORDER BY Id"; + using OleDbDataReader reader = command.ExecuteReader(); + for (int row = 1; row <= rowCount; row++) + { + Assert.True(reader.Read()); + Assert.Equal(row, reader.GetInt32(0)); + for (int column = 0; column < columns; column++) + Assert.Equal(expected[row][column], reader.GetString(column + 1)); + } + Assert.False(reader.Read()); + } + finally { TemporaryDatabase.Delete(path); } + } + [Fact] public void Access_seeks_a_libred_updated_primary_key() { diff --git a/test/LibRed.Core.AccessTests/UsageMapGrowthTests.cs b/test/LibRed.Core.AccessTests/UsageMapGrowthTests.cs index 32c27dfaa..015b9197c 100644 --- a/test/LibRed.Core.AccessTests/UsageMapGrowthTests.cs +++ b/test/LibRed.Core.AccessTests/UsageMapGrowthTests.cs @@ -14,6 +14,7 @@ namespace LibRed.Core.Tests; /// record in place (256-bit chunks) rather than switching to a reference map. LibRed does the same, so /// large tables keep working and Access still reads them. /// +[Collection(AceCollection.Name)] public class UsageMapGrowthTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/V1TailoringProbeTest.cs b/test/LibRed.Core.AccessTests/V1TailoringProbeTest.cs index 174c8682e..66fe57244 100644 --- a/test/LibRed.Core.AccessTests/V1TailoringProbeTest.cs +++ b/test/LibRed.Core.AccessTests/V1TailoringProbeTest.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // locales follow. This measures the departures rather than assuming either way. // // Opt-in via LIBRED_V1_TAILORING=1. +[Collection(AceCollection.Name)] public class V1TailoringProbeTest(ITestOutputHelper output) { /// diff --git a/test/LibRed.Core.AccessTests/VariableColumnHighWaterAccessTests.cs b/test/LibRed.Core.AccessTests/VariableColumnHighWaterAccessTests.cs index e13732990..dbb3d1994 100644 --- a/test/LibRed.Core.AccessTests/VariableColumnHighWaterAccessTests.cs +++ b/test/LibRed.Core.AccessTests/VariableColumnHighWaterAccessTests.cs @@ -24,6 +24,7 @@ namespace LibRed.Core.Tests; // column's slot index is abandoned when the column is dropped and the next one goes above it; a fixed // column's byte offset is reused. Nothing about one half predicts the other, and having them side by side // is the point: it is why writing the variable section by position looked reasonable. +[Collection(AceCollection.Name)] public class VariableColumnHighWaterAccessTests(ITestOutputHelper output) { // Drop a variable column from the MIDDLE. Dropping the last one would leave the high-water and the live diff --git a/test/LibRed.Core.AccessTests/WideIndexedSchemaChurnAccessTests.cs b/test/LibRed.Core.AccessTests/WideIndexedSchemaChurnAccessTests.cs new file mode 100644 index 000000000..ec2d058ae --- /dev/null +++ b/test/LibRed.Core.AccessTests/WideIndexedSchemaChurnAccessTests.cs @@ -0,0 +1,87 @@ +using System.Data.OleDb; +using LibRed; +using LibRed.Catalog; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// An ACE-oracle stress shape: a multi-page TDEF with split indexes and live rows is edited through both +/// metadata-only and full-rebuild paths. ACE must be able to read, seek, append and update the result. +/// +[Collection(AceCollection.Name)] +public class WideIndexedSchemaChurnAccessTests +{ + private const int ExtraColumns = 240; + private const int RowCount = 600; + + [Fact] + public void Ace_reads_and_extends_a_wide_indexed_table_after_schema_churn() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "wide-index-churn-"); + try + { + using (var database = JetDatabase.Open(path, readOnly: false)) + { + var columns = new List + { + new("Id", JetDataType.Int32, 4, IsFixedLength: true), + new("LookupKey", JetDataType.Text, 80, IsFixedLength: false), + new("Notes", JetDataType.Memo, 0, IsFixedLength: false), + }; + for (int i = 0; i < ExtraColumns; i++) + columns.Add(new ColumnSpec($"C{i}", JetDataType.Int32, 4, IsFixedLength: true)); + + database.CreateTable("WideIndexed", columns, primaryKey: ["Id"]); + database.CreateIndex("WideIndexed", "IX_LookupKey", [("LookupKey", false)], isUnique: true); + database.CreateIndex("WideIndexed", "IX_C0", [("C0", false)]); + + Table table = database.OpenTable("WideIndexed"); + for (int id = 1; id <= RowCount; id++) + { + var values = new object?[columns.Count]; + values[0] = id; + values[1] = $"key-{id:D4}"; + values[2] = id % 75 == 0 ? new string((char)('a' + id % 26), 5_000) : null; + values[3] = id * 10; + table.Insert(values); + } + + // Metadata-only edits leave id/variable-index gaps; the indexed retypes force full rebuilds. + Assert.True(database.AddColumn("WideIndexed", new ColumnSpec("Added", JetDataType.Int32, 4, IsFixedLength: true))); + Assert.True(database.AddColumn("WideIndexed", new ColumnSpec("AddedMemo", JetDataType.Memo, 0, IsFixedLength: false))); + Assert.True(database.DropColumn("WideIndexed", "C10")); + database.RenameColumn("WideIndexed", "LookupKey", "SeekKey"); + database.AlterColumn("WideIndexed", "SeekKey", new ColumnSpec("SeekKey", JetDataType.Text, 160, IsFixedLength: false)); + } + + using var connection = AceTestDatabase.Open(path); + AssertScalar(connection, "SELECT COUNT(*) FROM WideIndexed", RowCount); + AssertScalar(connection, "SELECT C0 FROM WideIndexed WHERE SeekKey = 'key-0420'", 4200); + AssertScalar(connection, "SELECT COUNT(*) FROM WideIndexed WHERE Added IS NULL", RowCount); + + using (var append = connection.CreateCommand()) + { + append.CommandText = "INSERT INTO WideIndexed (Id, SeekKey, C0, Added, AddedMemo) VALUES (701, 'key-0701', 7010, 8, 'ACE wrote this')"; + Assert.Equal(1, append.ExecuteNonQuery()); + } + using (var update = connection.CreateCommand()) + { + update.CommandText = "UPDATE WideIndexed SET SeekKey = 'key-0702', C0 = 7011 WHERE Id = 701"; + Assert.Equal(1, update.ExecuteNonQuery()); + } + AssertScalar(connection, "SELECT Id FROM WideIndexed WHERE SeekKey = 'key-0702'", 701); + AssertScalar(connection, "SELECT C0 FROM WideIndexed WHERE C0 = 7011", 7011); + AssertScalar(connection, "SELECT COUNT(*) FROM WideIndexed", RowCount + 1); + } + finally { TemporaryDatabase.Delete(path); } + } + + private static void AssertScalar(OleDbConnection connection, string sql, int expected) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + Assert.Equal(expected, Convert.ToInt32(command.ExecuteScalar())); + } +} diff --git a/test/LibRed.Core.AccessTests/WideMemoUsageMapAccessTests.cs b/test/LibRed.Core.AccessTests/WideMemoUsageMapAccessTests.cs index 9ff95836e..365a57563 100644 --- a/test/LibRed.Core.AccessTests/WideMemoUsageMapAccessTests.cs +++ b/test/LibRed.Core.AccessTests/WideMemoUsageMapAccessTests.cs @@ -17,6 +17,7 @@ namespace LibRed.Core.Tests; // ACE does not squeeze it in either - it spills, exactly as it does for the columns, leaving the full page // alone and putting the new index's map at row 0 of a page of its own. That rule was already in LibRed for // long-value columns in Create and AddColumn; InsertIndex was the one write path without it. +[Collection(AceCollection.Name)] public class WideMemoUsageMapAccessTests(ITestOutputHelper output) : TempDatabaseTest { private const int WideMemoColumns = 40; // more than the ~27 that fit alongside the table and index maps diff --git a/test/LibRed.Core.AccessTests/WideTableAccessTests.cs b/test/LibRed.Core.AccessTests/WideTableAccessTests.cs index f647148ba..59e9376a5 100644 --- a/test/LibRed.Core.AccessTests/WideTableAccessTests.cs +++ b/test/LibRed.Core.AccessTests/WideTableAccessTests.cs @@ -9,6 +9,7 @@ namespace LibRed.Core.Tests; /// A table wide enough that its TDEF spans continuation pages (the owned-types / proxy shape). LibRed writes /// the definition split across pages; Access opens it, reports every column, and round-trips a row. /// +[Collection(AceCollection.Name)] public class WideTableAccessTests { private static OleDbConnection OpenOleDb(string path) => AceTestDatabase.Open(path); diff --git a/test/LibRed.Core.AccessTests/WideTableUsageMapTests.cs b/test/LibRed.Core.AccessTests/WideTableUsageMapTests.cs index 0c46c8205..8861285b4 100644 --- a/test/LibRed.Core.AccessTests/WideTableUsageMapTests.cs +++ b/test/LibRed.Core.AccessTests/WideTableUsageMapTests.cs @@ -26,6 +26,7 @@ namespace LibRed.Core.Tests; /// Access must be able to read the result — that is what makes the layout byte-faithful rather than /// merely self-consistent. /// +[Collection(AceCollection.Name)] public class WideTableUsageMapTests { private const int Columns = 255; diff --git a/test/LibRed.Core.AccessTests/xunit.runner.json b/test/LibRed.Core.AccessTests/xunit.runner.json deleted file mode 100644 index 08c512b3d..000000000 --- a/test/LibRed.Core.AccessTests/xunit.runner.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "parallelizeTestCollections": false -} diff --git a/test/LibRed.Core.Tests/CalculatedEvaluatorTests.cs b/test/LibRed.Core.Tests/CalculatedEvaluatorTests.cs index 6e56a0a43..c92803f7e 100644 --- a/test/LibRed.Core.Tests/CalculatedEvaluatorTests.cs +++ b/test/LibRed.Core.Tests/CalculatedEvaluatorTests.cs @@ -37,9 +37,22 @@ public void Evaluates_arithmetic(string expression, double expected) [InlineData("\"a\" & \"b\"", "ab")] [InlineData("\"n=\" & 1", "n=1")] [InlineData("\"a\" + \"b\"", "ab")] // '+' concatenates when BOTH sides are text + [InlineData("'a' + ' ' + 'b'", "a b")] // single quotes too, as ACE accepts them (verified through DAO) + [InlineData("'it''s' & \"x\"", "it'sx")] + [InlineData("\"say \"\"hi\"\"\" & 'x'", "say \"hi\"x")] public void Evaluates_concatenation(string expression, string expected) => Assert.Equal(expected, Eval(expression)); + [Fact] + public void Single_quoted_text_joins_columns() => + Assert.Equal("x y", Eval("FirstName + ' ' + LastName", ("FirstName", "x"), ("LastName", "y"))); + + [Theory] + [InlineData("'abc")] + [InlineData("\"abc'")] + public void An_unclosed_quote_is_an_error(string expression) => + Assert.Throws(() => CalculatedExpression.Parse(expression)); + // The asymmetry MSDN uses to introduce the feature: '+' propagates Null, '&' swallows it. Reversing // them leaves a stray space in the classic full-name expression, which nothing would ever flag. [Fact] diff --git a/test/LibRed.Core.Tests/DropTableReclamationTests.cs b/test/LibRed.Core.Tests/DropTableReclamationTests.cs new file mode 100644 index 000000000..4e949b6ac --- /dev/null +++ b/test/LibRed.Core.Tests/DropTableReclamationTests.cs @@ -0,0 +1,107 @@ +using LibRed; +using LibRed.Catalog; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// Dropping a table must return its long-value pages, not just its data pages. A Memo/OLE column owns its +/// LVAL pages through a per-column usage map whose pointer lives in the TDEF keyed by column id — those pages +/// are absent from the table's own data-page map, so a drop that frees only the data pages strands the entire +/// content of the table. Measured against ACE before the fix: dropping a 60-row memo table returned 123 pages +/// through ACE and 2 through LibRed, and a subsequent refill grew the file by 119 pages instead of reusing it. +/// +public class DropTableReclamationTests +{ + [Fact] + public void Dropping_a_memo_table_frees_its_long_value_pages() + { + string path = TemporaryDatabase.CreatePath("libred_drop_lval_"); + try + { + DatabaseCreator.CreateEmpty(path); + string big = new('x', 3000); // far past the inline threshold, so each value owns LVAL pages + + int filled, freeBefore, tdefPage; + byte[] releasedTdef; + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("Victim", [ + new ColumnSpec("Id", JetDataType.Int32, 4, true), + new ColumnSpec("M", JetDataType.Memo, 0, false)]); + db.Catalog.Invalidate(); + + Table victim = db.OpenTable("Victim"); + for (int i = 0; i < 60; i++) victim.Insert([i, big]); + + filled = victim.Channel.PageCount; + tdefPage = db.Catalog.FindTable("Victim")!.DefinitionPage; + freeBefore = FreePages(db); + db.DropTable("Victim"); + + // Held until this handle closes, as ACE holds a dropped table's pages. + Assert.Equal(freeBefore, FreePages(db)); + + releasedTdef = new byte[victim.Channel.PageSize]; + victim.Channel.ReadPage(tdefPage, releasedTdef); + } + + int freedPages; + using (var db = JetDatabase.Open(path)) + freedPages = FreePages(db) - freeBefore; + + // Access marks the released definition page and leaves the rest of it alone — measured, exactly + // one byte of the 4,096 changes across an ACE drop. The pages it frees alongside keep their types. + Assert.Equal((byte)LibRed.Pages.PageType.ReleasedTableDefinition, releasedTdef[0]); + + // 60 values of 3,000 characters is ~360 KB of UTF-16, well over a hundred 4 KB pages. Before the + // fix this was 2 — the single data page plus the TDEF. + Assert.True(freedPages > 100, + $"dropping the table returned only {freedPages} pages to the global free map"); + + // The functional consequence: refilling reuses the space instead of extending the file. + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("Keeper", [ + new ColumnSpec("Id", JetDataType.Int32, 4, true), + new ColumnSpec("M", JetDataType.Memo, 0, false)]); + db.Catalog.Invalidate(); + + Table keeper = db.OpenTable("Keeper"); + for (int i = 0; i < 60; i++) keeper.Insert([i, big]); + + Assert.True(keeper.Channel.PageCount <= filled + 8, + $"refilling grew the file from {filled} to {keeper.Channel.PageCount} pages, so the " + + "dropped table's pages were not reused"); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + /// How many pages the global free-pages map (page 1, row 0) currently calls free. + private static int FreePages(JetDatabase db) + { + var channel = db.OpenTable("MSysObjects").Channel; + int free = 0; + for (int page = 2; page < channel.PageCount; page++) + if (IsFree(channel, page)) free++; + return free; + } + + private static bool IsFree(LibRed.IO.PageChannel channel, int page) + { + var buffer = new byte[channel.PageSize]; + channel.ReadPage(1, buffer); + var holder = new LibRed.Pages.DataPage(); + holder.Read(channel.ReadPage(1), channel.Format); + if (holder.RowCount < 1) return false; + var slot = holder.Rows[0]; + if (buffer[slot.Offset] != 0x00) return false; // inline form only; a tiny file never grows past it + int start = BitConverter.ToInt32(buffer, slot.Offset + 1); + int bit = page - start; + int index = slot.Offset + 5 + bit / 8; + if (bit < 0 || index >= slot.Offset + slot.Length) return false; + return (buffer[index] & (1 << (bit % 8))) != 0; + } +} diff --git a/test/LibRed.Core.Tests/GlobalMapPointerTests.cs b/test/LibRed.Core.Tests/GlobalMapPointerTests.cs new file mode 100644 index 000000000..3c68b9c27 --- /dev/null +++ b/test/LibRed.Core.Tests/GlobalMapPointerTests.cs @@ -0,0 +1,206 @@ +using System.Buffers.Binary; +using LibRed.Formats; +using LibRed.IO; +using LibRed.Pages; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +// Page 0's 0x18 and 0x1C are [row][page] pointers to the global free-pages and released-pages usage maps. ACE +// follows both wherever they point, never allocates a released page, and a file whose pointers are wrong is +// corrupt to it (docs/format/page-05-usage-maps.md §9.1). Northwind's maps are the ACE layout: page 1 row 0 free +// (pages 310 and 329 inside the file, 353..511 past its end), page 1 row 1 released and empty. +public class GlobalMapPointerTests +{ + private const int NorthwindPages = 353; + + [Fact] + public void Page_zero_decodes_the_global_map_pointers() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-decode-"); + try + { + using var db = JetDatabase.Open(path); + Assert.Equal((0, 1), db.DefinitionPage.FreePagesMap); + Assert.Equal((1, 1), db.DefinitionPage.ReleasedPagesMap); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void Allocation_follows_page_zero_to_maps_on_another_page() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-moved-"); + try + { + // Copy page 1 to a new page 353, mark 353 itself used in the copy, and aim both pointers at it. + byte[] page1 = ReadPage(path, 1); + byte[] moved = (byte[])page1.Clone(); + SetMapBit(moved, row: 0, page: NorthwindPages, set: false); + AppendPage(path, moved); + WritePointer(path, JetFormatBase.FreePagesMapPointerOffset, row: 0, page: NorthwindPages); + WritePointer(path, JetFormatBase.ReleasedPagesMapPointerOffset, row: 1, page: NorthwindPages); + // Poison page 1's stale free map: were it still read, the next allocation would be page 400. + byte[] stale = (byte[])page1.Clone(); + for (int p = 0; p < 512; p++) SetMapBit(stale, row: 0, page: p, set: p == 400); + WritePage(path, 1, stale); + + using (var channel = PageChannel.Open(path, readOnly: false)) + { + var allocator = new PageAllocator(channel); + Assert.Equal(310, allocator.Allocate()); + Assert.Equal(329, allocator.Allocate()); + Assert.Equal(354, allocator.Allocate()); // 353 is the map's own page, and used + } + + Assert.Equal(stale, ReadPage(path, 1)); // never read or written + byte[] after = ReadPage(path, NorthwindPages); + Assert.False(MapBit(after, 0, 310)); + Assert.False(MapBit(after, 0, 329)); + Assert.False(MapBit(after, 0, 354)); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void Allocation_skips_pages_set_in_the_released_map() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-released-"); + try + { + byte[] page1 = ReadPage(path, 1); + SetMapBit(page1, row: 1, page: 310, set: true); + SetMapBit(page1, row: 1, page: 329, set: true); + WritePage(path, 1, page1); + + using (var channel = PageChannel.Open(path, readOnly: false)) + { + var allocator = new PageAllocator(channel); + Assert.Equal(NorthwindPages, allocator.Allocate()); // past the two released pages + Assert.Equal(NorthwindPages + 1, allocator.Allocate()); + } + + byte[] after = ReadPage(path, 1); + Assert.True(MapBit(after, 0, 310)); // still free: released, not taken + Assert.True(MapBit(after, 0, 329)); + Assert.True(MapBit(after, 1, 310)); // and still released + Assert.True(MapBit(after, 1, 329)); + } + finally { TemporaryDatabase.Delete(path); } + } + + // Released pages can sit past the end of the file (a session that claimed them never wrote them). The file + // stays contiguous, so they are materialized — but the allocation is the first page after them. + [Fact] + public void Released_pages_at_the_end_of_the_file_are_materialized_but_not_allocated() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-frontier-"); + try + { + byte[] page1 = ReadPage(path, 1); + foreach (int p in new[] { 310, 329, NorthwindPages, NorthwindPages + 1 }) + SetMapBit(page1, row: 1, page: p, set: true); + WritePage(path, 1, page1); + + using (var channel = PageChannel.Open(path, readOnly: false)) + { + Assert.Equal(NorthwindPages + 2, new PageAllocator(channel).Allocate()); + Assert.Equal(NorthwindPages + 3, channel.PageCount); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + public static TheoryData InvalidPointers => new() + { + // name, free row, free page, released row, released page + { "free map past the end of the file", 0, NorthwindPages, 1, 1 }, + { "released map past the end of the file", 0, 1, 1, NorthwindPages }, + { "both maps on the same record", 0, 1, 0, 1 }, + { "free map on a TDEF page", 0, 2, 1, 1 }, + { "released map on a row the page does not have", 0, 1, 5, 1 }, + { "free map on page 0", 0, 0, 1, 1 }, + }; + + [Theory] + [MemberData(nameof(InvalidPointers))] + public void A_writable_open_refuses_invalid_global_map_pointers( + string name, int freeRow, int freePage, int releasedRow, int releasedPage) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "globalptr-invalid-"); + try + { + WritePointer(path, JetFormatBase.FreePagesMapPointerOffset, freeRow, freePage); + WritePointer(path, JetFormatBase.ReleasedPagesMapPointerOffset, releasedRow, releasedPage); + + var error = Assert.Throws(() => JetDatabase.Open(path, readOnly: false)); + Assert.Contains("global", error.Message, StringComparison.OrdinalIgnoreCase); + + // A read-only open never allocates, and reads the file as ACE does. + using var readOnly = JetDatabase.Open(path, readOnly: true); + Assert.NotNull(readOnly.Catalog.FindTable("Customers")); + _ = name; + } + finally { TemporaryDatabase.Delete(path); } + } + + // ---------------------------------------------------------------- helpers + + private static void WritePointer(string path, int offset, int row, int page) + { + ReadOnlySpan mask = JetFormatBase.PageZeroHeaderMask; + byte[] value = BitConverter.GetBytes((uint)(page << 8 | row)); + for (int i = 0; i < 4; i++) value[i] ^= mask[offset - JetFormatBase.PageZeroHeaderMaskStart + i]; + using var s = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + s.Position = offset; + s.Write(value); + } + + private static byte[] ReadPage(string path, int page) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + var bytes = new byte[4096]; + s.Position = page * 4096L; + s.ReadExactly(bytes); + return bytes; + } + + private static void WritePage(string path, int page, byte[] bytes) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + s.Position = page * 4096L; + s.Write(bytes); + } + + private static void AppendPage(string path, byte[] bytes) + { + using var s = new FileStream(path, FileMode.Append, FileAccess.Write); + s.Write(bytes); + } + + /// The bitmap byte and bit for in the inline map at + /// of a usage-map holder page. + private static (int Byte, int Bit) Locate(byte[] holder, int row, int page) + { + int offset = BinaryPrimitives.ReadUInt16LittleEndian(holder.AsSpan(14 + row * 2)) & 0x1FFF; + int end = row == 0 ? 4096 : BinaryPrimitives.ReadUInt16LittleEndian(holder.AsSpan(14 + (row - 1) * 2)) & 0x1FFF; + Assert.Equal(0x00, holder[offset]); // inline + int bit = page - BinaryPrimitives.ReadInt32LittleEndian(holder.AsSpan(offset + 1)); + Assert.InRange(bit, 0, (end - offset - 5) * 8 - 1); + return (offset + 5 + bit / 8, bit % 8); + } + + private static void SetMapBit(byte[] holder, int row, int page, bool set) + { + (int b, int bit) = Locate(holder, row, page); + if (set) holder[b] |= (byte)(1 << bit); + else holder[b] &= (byte)~(1 << bit); + } + + private static bool MapBit(byte[] holder, int row, int page) + { + (int b, int bit) = Locate(holder, row, page); + return (holder[b] & (1 << bit)) != 0; + } +} diff --git a/test/LibRed.Core.Tests/GlobalReferenceFreeMapTests.cs b/test/LibRed.Core.Tests/GlobalReferenceFreeMapTests.cs index eeeb07e4a..1696c165c 100644 --- a/test/LibRed.Core.Tests/GlobalReferenceFreeMapTests.cs +++ b/test/LibRed.Core.Tests/GlobalReferenceFreeMapTests.cs @@ -29,12 +29,14 @@ public void Allocate_and_free_through_a_reference_type_global_map() creationDays: 45000).CopyTo(file, 0); // Page 1 — a data page whose row 0 is a reference-type global free map: slot 0 → bitmap page 2, - // slot 1 → bitmap page 3. (The 69-byte record is packed at the page end, as ACE packs rows.) + // slot 1 → bitmap page 3. (The 69-byte record is packed at the page end, as ACE packs rows.) Row 1 is the + // released-pages map page 0 points at (0x1C), empty, as every real file carries it. int p1 = pageSize; file[p1] = 0x01; // page type: data page - BinaryPrimitives.WriteUInt16LittleEndian(file.AsSpan(p1 + format.DataRowCountOffset, 2), 1); + BinaryPrimitives.WriteUInt16LittleEndian(file.AsSpan(p1 + format.DataRowCountOffset, 2), 2); int mapOffset = pageSize - 69; BinaryPrimitives.WriteUInt16LittleEndian(file.AsSpan(p1 + format.DataRowDirectoryOffset, 2), (ushort)mapOffset); + BinaryPrimitives.WriteUInt16LittleEndian(file.AsSpan(p1 + format.DataRowDirectoryOffset + 2, 2), (ushort)(mapOffset - 69)); file[p1 + mapOffset] = 0x01; // reference map type BinaryPrimitives.WriteInt32LittleEndian(file.AsSpan(p1 + mapOffset + 1 + 0 * 4, 4), 2); // slot 0 → page 2 BinaryPrimitives.WriteInt32LittleEndian(file.AsSpan(p1 + mapOffset + 1 + 1 * 4, 4), 3); // slot 1 → page 3 diff --git a/test/LibRed.Core.Tests/LockManagerTests.cs b/test/LibRed.Core.Tests/LockManagerTests.cs index f50f16a2d..dc54f158c 100644 --- a/test/LibRed.Core.Tests/LockManagerTests.cs +++ b/test/LibRed.Core.Tests/LockManagerTests.cs @@ -5,6 +5,10 @@ namespace LibRed.Core.Tests; public class LockManagerTests { + // How long to wait for the other thread to get in when it should. Only a failing test waits this long, so it is + // generous: a busy CI runner can take seconds to schedule the thread. + private const int Patience = 30_000; + [Fact] public void Multiple_readers_hold_the_same_page_concurrently() { @@ -15,9 +19,9 @@ public void Multiple_readers_hold_the_same_page_concurrently() try { var secondEntered = new ManualResetEventSlim(); - var t = Task.Run(() => { m.EnterShared(5); m.ExitShared(5); secondEntered.Set(); }); - Assert.True(secondEntered.Wait(2000)); // not blocked by the first reader - t.Wait(2000); + Thread t = StartThread(() => { m.EnterShared(5); m.ExitShared(5); secondEntered.Set(); }); + Assert.True(secondEntered.Wait(Patience)); // not blocked by the first reader + t.Join(Patience); } finally { m.ExitShared(5); } } @@ -28,13 +32,24 @@ public void A_writer_blocks_a_reader_of_the_same_page_until_released() var m = new MonitorLockManager(); m.EnterExclusive(5); + var readerStarted = new ManualResetEventSlim(); var readerEntered = new ManualResetEventSlim(); - var reader = Task.Run(() => { m.EnterShared(5); m.ExitShared(5); readerEntered.Set(); }); + Thread reader = StartThread(() => { readerStarted.Set(); m.EnterShared(5); m.ExitShared(5); readerEntered.Set(); }); + Assert.True(readerStarted.Wait(Patience)); Assert.False(readerEntered.Wait(250)); // the writer holds the page, so the reader can't enter m.ExitExclusive(5); - Assert.True(readerEntered.Wait(2000)); // released — the reader proceeds - reader.Wait(2000); + Assert.True(readerEntered.Wait(Patience)); // released — the reader proceeds + reader.Join(Patience); + } + + // A thread of its own rather than Task.Run: the thread pool is shared with every test running in parallel, and + // can leave a queued item waiting longer than any sensible timeout. + private static Thread StartThread(Action body) + { + var thread = new Thread(() => body()) { IsBackground = true }; + thread.Start(); + return thread; } [Fact] diff --git a/test/LibRed.Core.Tests/MultiPageDefinitionTests.cs b/test/LibRed.Core.Tests/MultiPageDefinitionTests.cs index a1531a605..d3be3683f 100644 --- a/test/LibRed.Core.Tests/MultiPageDefinitionTests.cs +++ b/test/LibRed.Core.Tests/MultiPageDefinitionTests.cs @@ -45,4 +45,42 @@ public void Index_that_overflows_the_tdef_page_spills_to_a_continuation_and_roun } finally { TemporaryDatabase.Delete(path); } } + + // The chain holds the definition and then its 8-byte reserve, which spills onto a page of its own when it does + // not fit (docs/format/page-02a-tdef.md §3.2). The free space of each page is ACE's, measured for the same + // tables: 115 LONG columns make 4,086 bytes, each character on the first name adds two, 232 columns make 8,181. + [Theory] + [InlineData(115, 1, new[] { 0 })] + [InlineData(115, 2, new[] { 0, 4086 })] // 4,090 bytes: the continuation holds two reserve bytes + [InlineData(115, 5, new[] { 0, 4080 })] // 4,096 bytes: it holds the whole reserve and no definition + [InlineData(115, 6, new[] { 0, 4078 })] + [InlineData(232, 0, new[] { 0, 0, 4083 })] // 8,181 bytes: a third page for five reserve bytes + public void A_definition_near_a_page_boundary_spills_its_reserve_and_reads_back(int columns, int longerName, int[] free) + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "spill-"); + try + { + string[] names = ["Id", "c001" + new string('x', longerName), .. Enumerable.Range(2, columns - 2).Select(i => $"c{i:D3}")]; + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable("L", names.Select(n => new ColumnSpec(n, JetDataType.Int32, 4, IsFixedLength: true)).ToList()); + db.OpenTable("L").Insert([7, .. Enumerable.Repeat(null, columns - 1)]); + } + + using (var ch = PageChannel.Open(path, readOnly: true)) + { + var chainFree = new List(); + for (int page = new JetCatalog(ch).FindTable("L")!.DefinitionPage; page != 0; page = ch.ReadPage(page).ReadInt32(ch.Format.TdefNextPageOffset)) + chainFree.Add(ch.ReadPage(page).ReadUInt16(ch.Format.TdefFreeSpaceOffset)); + Assert.Equal(free, chainFree); + } + + using (var db = JetDatabase.Open(path)) + { + Assert.Equal(names, db.Catalog.FindTable("L")!.Columns.Select(c => c.Name)); + Assert.Equal(7, db.OpenTable("L").Rows().Single()[0]); + } + } + finally { TemporaryDatabase.Delete(path); } + } } diff --git a/test/LibRed.Core.Tests/RelationshipObjectTests.cs b/test/LibRed.Core.Tests/RelationshipObjectTests.cs new file mode 100644 index 000000000..b0dbf5c6f --- /dev/null +++ b/test/LibRed.Core.Tests/RelationshipObjectTests.cs @@ -0,0 +1,97 @@ +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Core.Tests; + +/// +/// Every relationship is also a type-8 MSysObjects object in the Relationships container, with two +/// MSysACEs rows, as ACE records it (docs/format/system-catalog.md). +/// +public class RelationshipObjectTests : TempDatabaseTest +{ + private static readonly RelationshipSpec Fk = new("fkCP", "P", [("PId", "Id")], IsEnforced: true, CascadeUpdate: false, CascadeDelete: false); + + private static string Database() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "relobj-"); + using var db = JetDatabase.Open(path, readOnly: false); + db.CreateTable("P", [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true)], primaryKey: ["Id"]); + db.CreateTable("C", [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), new ColumnSpec("PId", JetDataType.Int32, 4, IsFixedLength: true)]); + return path; + } + + /// MSysObjects rows as column-name dictionaries. + private static List> Objects(JetDatabase db) => Rows(db, "MSysObjects"); + + private static List> Rows(JetDatabase db, string table) + { + IReadOnlyList columns = db.Catalog.FindTable(table)!.Columns; + return db.OpenTable(table).Rows().Select(r => columns.ToDictionary(c => c.Name, c => r[c.Index])).ToList(); + } + + [Fact] + public void A_relationship_is_recorded_as_a_type_8_object_with_two_permission_rows() + { + string path = Database(); + int highest; + using (var db = JetDatabase.Open(path, readOnly: false)) + { + highest = Objects(db).Select(o => Convert.ToInt32(o["Id"])).Where(id => id < 0).Max(); + db.AddForeignKey("C", Fk); + } + + using var check = JetDatabase.Open(path); + Dictionary fk = Objects(check).Single(o => Equals(o["Name"], "fkCP")); + Assert.Equal((short)8, fk["Type"]); + Assert.Equal(0x0F000003, fk["ParentId"]); + Assert.Equal(0, fk["Flags"]); + Assert.Equal(new byte[] { 0x69, 0x0C }, fk["Owner"]); + Assert.Equal(highest + 1, fk["Id"]); + Assert.Null(fk["LvProp"]); + + var aces = Rows(check, "MSysACEs").Where(a => Equals(a["ObjectId"], fk["Id"])).ToList(); + Assert.Equal(2, aces.Count); + Assert.Equal(0xF00FE, aces.Single(a => ((byte[])a["SID"]!)[0] == 0x69)["ACM"]); + Assert.Equal(0xFFFFF, aces.Single(a => ((byte[])a["SID"]!)[0] == 0x68)["ACM"]); + } + + [Fact] + public void Dropping_the_relationship_removes_its_object_and_the_next_one_takes_its_id() + { + string path = Database(); + using var db = JetDatabase.Open(path, readOnly: false); + db.AddForeignKey("C", Fk); + int id = Convert.ToInt32(Objects(db).Single(o => Equals(o["Name"], "fkCP"))["Id"]); + + Assert.True(db.DropConstraint("C", "fkCP")); + Assert.DoesNotContain(Objects(db), o => Equals(o["Name"], "fkCP")); + Assert.DoesNotContain(Rows(db, "MSysACEs"), a => Equals(a["ObjectId"], id)); + + db.AddForeignKey("C", Fk with { Name = "fkAgain" }); + Assert.Equal(id, Objects(db).Single(o => Equals(o["Name"], "fkAgain"))["Id"]); + } + + [Fact] + public void Dropping_the_referencing_table_removes_the_relationship_object() + { + string path = Database(); + using var db = JetDatabase.Open(path, readOnly: false); + db.AddForeignKey("C", Fk); + + Assert.True(db.DropTable("C")); + Assert.DoesNotContain(Objects(db), o => Equals(o["Name"], "fkCP")); + } + + [Fact] + public void A_relationship_may_share_a_tables_name_but_not_another_relationships() + { + string path = Database(); + using var db = JetDatabase.Open(path, readOnly: false); + db.AddForeignKey("C", Fk with { Name = "P" }); + Assert.Contains(Objects(db), o => Equals(o["Name"], "P") && Equals(o["Type"], (short)8)); + + db.CreateTable("D", [new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true), new ColumnSpec("PId", JetDataType.Int32, 4, IsFixedLength: true)]); + var refused = Assert.Throws(() => db.AddForeignKey("D", Fk with { Name = "P" })); + Assert.Equal("There is already a relationship named 'P' in the current database.", refused.Message); + } +} diff --git a/test/LibRed.Core.Tests/ReleaseAtCloseTests.cs b/test/LibRed.Core.Tests/ReleaseAtCloseTests.cs new file mode 100644 index 000000000..9377d1e92 --- /dev/null +++ b/test/LibRed.Core.Tests/ReleaseAtCloseTests.cs @@ -0,0 +1,223 @@ +using System.Buffers.Binary; +using LibRed.Catalog; +using LibRed.IO; +using LibRed.Storage; +using Xunit; + +namespace LibRed.Core.Tests; + +// ACE holds the pages a session frees — a deleted row's long values, a dropped index, a dropped table — until the +// session closes, then returns them and anything in the global released-pages map to the global free map and +// clears the released map, lengthening it to cover the highest page released (docs/format/page-05-usage-maps.md +// §9.1). The long value an UPDATE replaces is the exception: its pages are free at once. Northwind's maps are page 1 +// rows 0 (free: 310 and 329 inside the file) and 1 (released, empty). +public class ReleaseAtCloseTests +{ + [Fact] + public void Released_pages_stay_unreusable_until_close_then_become_free() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-close-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + var allocator = new PageAllocator(channel); + allocator.Release(300); + allocator.Release(301); + + Assert.False(MapBit(channel, 0, 300)); + Assert.False(MapBit(channel, 0, 301)); + Assert.Equal(310, allocator.Allocate()); // the free pages, never the released ones + Assert.Equal(329, allocator.Allocate()); + Assert.Equal(353, allocator.Allocate()); + } + + using (var db = JetDatabase.Open(path)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + Assert.True(MapBit(channel, 0, 300)); + Assert.True(MapBit(channel, 0, 301)); + Assert.False(MapBit(channel, 0, 310)); + Assert.All(ReleasedBits(channel), b => Assert.Equal(0, b)); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void A_rolled_back_release_frees_nothing_and_a_savepoint_rollback_keeps_the_earlier_ones() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-rollback-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + var allocator = new PageAllocator(channel); + + channel.BeginTransaction(); + allocator.Release(300); + channel.RollbackTransaction(); + + channel.BeginTransaction(); + allocator.Release(301); + Savepoint savepoint = channel.CreateSavepoint(); + allocator.Release(302); + channel.RollbackToSavepoint(savepoint); + channel.CommitTransaction(); + + channel.BeginTransaction(); + allocator.Release(303); + // left open: the close discards it + } + + using (var db = JetDatabase.Open(path)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + Assert.False(MapBit(channel, 0, 300)); + Assert.True(MapBit(channel, 0, 301)); + Assert.False(MapBit(channel, 0, 302)); + Assert.False(MapBit(channel, 0, 303)); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void The_close_lengthens_the_released_map_to_cover_the_highest_page_released() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-length-"); + try + { + int before, highest; + using (var db = JetDatabase.Open(path, readOnly: false)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + var allocator = new PageAllocator(channel); + before = ReleasedBits(channel).Length; + for (int i = 0; i < 400; i++) allocator.Allocate(); + highest = channel.PageCount - 1; + allocator.Release(highest); + } + + using (var db = JetDatabase.Open(path)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + // 5-byte header, then the bitmap to the highest page in whole 4-byte words. + int expected = ((highest / 8 + 1) + 3) / 4 * 4; + Assert.True(expected > before); + Assert.Equal(expected, ReleasedBits(channel).Length); + Assert.All(ReleasedBits(channel), b => Assert.Equal(0, b)); + Assert.True(MapBit(channel, 0, highest)); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void Pages_already_in_the_released_map_are_merged_by_a_close_that_wrote_but_not_by_an_idle_one() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-merge-"); + try + { + byte[] page1 = ReadPage(path, 1); + SetReleasedBit(page1, 300); + WritePage(path, 1, page1); + + using (JetDatabase.Open(path, readOnly: false)) { } + Assert.Equal(page1, ReadPage(path, 1)); // an idle writable close changes nothing + + using (var db = JetDatabase.Open(path, readOnly: false)) + db.CreateTable("Wrote", [new("Id", JetDataType.Int32, 4, IsFixedLength: true)]); + + using (var db = JetDatabase.Open(path)) + { + PageChannel channel = db.OpenTable("MSysObjects").Channel; + Assert.True(MapBit(channel, 0, 300)); + Assert.All(ReleasedBits(channel), b => Assert.Equal(0, b)); + } + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void Dropping_an_index_holds_its_root_but_a_memo_update_frees_the_old_chain_at_once() + { + string path = TemporaryDatabase.CopyPath(TestDatabases.NorthwindAccdb, "release-paths-"); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + db.CreateTable("Paths", + [new("Id", JetDataType.Int32, 4, IsFixedLength: true), + new("M", JetDataType.Memo, 0, IsFixedLength: false)], + primaryKey: ["Id"]); + Table table = db.OpenTable("Paths"); + table.Insert([1, new string('a', 20000)]); + PageChannel channel = table.Channel; + + int root = db.Catalog.FindTable("Paths")!.Indexes.Single().RootPage; + db.DropIndex("Paths", db.Catalog.FindTable("Paths")!.Indexes.Single().Name); + Assert.False(MapBit(channel, 0, root)); + + table = db.OpenTable("Paths"); + (RowId id, object?[] values) = table.Rows().WithIds().Single(); + // The old chain is free at once, so the new value lands on it and the file does not grow. + int pagesBefore = channel.PageCount; + values[1] = new string('b', 20000); + table.Update(id, values, new HashSet { 1 }); + Assert.Equal(pagesBefore, channel.PageCount); + } + finally { TemporaryDatabase.Delete(path); } + } + + // ---------------------------------------------------------------- helpers + + private static (int Offset, int Length) Record(ReadOnlySpan page1, int row) + { + int offset = BinaryPrimitives.ReadUInt16LittleEndian(page1[(14 + row * 2)..]) & 0x1FFF; + int end = row == 0 ? page1.Length : BinaryPrimitives.ReadUInt16LittleEndian(page1[(14 + (row - 1) * 2)..]) & 0x1FFF; + Assert.Equal(0x00, page1[offset]); // inline + return (offset, end - offset); + } + + private static bool MapBit(PageChannel channel, int row, int page) + { + byte[] page1 = channel.ReadPage(1).Span.ToArray(); + (int offset, int length) = Record(page1, row); + int bit = page - BinaryPrimitives.ReadInt32LittleEndian(page1.AsSpan(offset + 1)); + if (bit < 0 || bit / 8 >= length - 5) return false; + return (page1[offset + 5 + bit / 8] & (1 << (bit % 8))) != 0; + } + + private static byte[] ReleasedBits(PageChannel channel) + { + byte[] page1 = channel.ReadPage(1).Span.ToArray(); + (int offset, int length) = Record(page1, 1); + return page1.AsSpan(offset + 5, length - 5).ToArray(); + } + + private static void SetReleasedBit(byte[] page1, int page) + { + (int offset, int length) = Record(page1, 1); + int bit = page - BinaryPrimitives.ReadInt32LittleEndian(page1.AsSpan(offset + 1)); + Assert.InRange(bit, 0, (length - 5) * 8 - 1); + page1[offset + 5 + bit / 8] |= (byte)(1 << (bit % 8)); + } + + private static byte[] ReadPage(string path, int page) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + var bytes = new byte[4096]; + s.Position = page * 4096L; + s.ReadExactly(bytes); + return bytes; + } + + private static void WritePage(string path, int page, byte[] bytes) + { + using var s = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + s.Position = page * 4096L; + s.Write(bytes); + } +} diff --git a/test/LibRed.Engine.AccessTests/AceCoreApiShapeProbeTests.cs b/test/LibRed.Engine.AccessTests/AceCoreApiShapeProbeTests.cs new file mode 100644 index 000000000..87b43ec78 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/AceCoreApiShapeProbeTests.cs @@ -0,0 +1,255 @@ +using LibRed; +using LibRed.Catalog; +using LibRed.Data; +using LibRed.Formats; +using Xunit; + +namespace LibRed.Engine.Tests; + +// The Core-API arm of the write-validity work. The sweep drives LibRed through its SQL front door, so every +// column it builds has already passed AccessTypeMapper — where the width, precision and scale caps live. This +// goes in the other door, the one JetVersion.cs warns about: a ColumnSpec straight to the writer. +// +// TdefBuilder already validates column count, names, ids, the 510-byte field cap and the widest record, and +// EnsureStorable covers version-gated types. This asks what is left OVER those guards. +// +// The contract is NOT "every shape must be accepted" — it is that whether LibRed accepts or refuses the spec, +// the file left behind must be one ACE accepts. A refusal does not end the test: several shapes are refused by +// the row ENCODER rather than the declaration ("Column 'V' encoded to 4 bytes, expected 1"), so the malformed +// table was already committed and only the INSERT failed. The ladder runs either way. +// +// A named catalogue rather than random specs: the space is small and enumerable, and a random walk would +// regenerate the same few cases while naming none of them. +[Collection(AceCollection.Name)] +public class AceCoreApiShapeProbeTests(ITestOutputHelper output) : TempDatabaseTest +{ + /// The unconstrained table rung 4 writes into, created alongside every shape. + private const string AceWriteTarget = "AceTarget"; + + public static TheoryData ShapeNames => [.. Shapes.Keys]; + + [Theory] + [MemberData(nameof(ShapeNames))] + public void Either_libred_refuses_the_shape_or_ace_accepts_the_file(string shape) + { + Shape definition = Shapes[shape]; + output.WriteLine($"{shape}: {definition.Why}"); + + string path = TemporaryDatabase.CreatePath("libred-coreshape-"); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}", collation: null, version: definition.Version); + + using (var db = JetDatabase.Open(path, readOnly: false)) + { + db.CreateTable(AceWriteTarget, + [ + new ColumnSpec("K", JetDataType.Int32, 4, IsFixedLength: true), + new ColumnSpec("V", JetDataType.Text, 100, IsFixedLength: false), + ]); + + try + { + definition.Apply(db); + output.WriteLine(" LibRed ACCEPTED it."); + } + catch (Exception ex) + { + // A guard fired — better, but not the end of the test: an encoder-level refusal leaves the + // malformed table committed, so the ladder still has to see what it left. + output.WriteLine( + $" LibRed REFUSED it: {ex.GetType().Name}: {AceValidityLadder.Flatten(ex.Message)}"); + output.WriteLine(" (checking the file the refusal left behind)"); + } + } + + AceValidityLadder.Verdict verdict = AceValidityLadder.Check(path, AceWriteTarget); + foreach (string typedOnly in verdict.TypedOnly) output.WriteLine($" (typed-read only) {typedOnly}"); + + Assert.True(verdict.Finding is null, + $"The '{shape}' specification left a file ACE will not take.\n" + + $" {definition.Why}\n {verdict.Finding}"); + } + finally { TemporaryDatabase.Delete(path); } + } + + private sealed record Shape(string Why, Action Apply, JetVersion Version = JetVersion.Version12_2007); + + private static readonly Dictionary Shapes = new() + { + // ---- Precision and scale, which nothing checked until these shapes found it: the width check covers + // only the fixed 17 bytes. TdefBuilder.ValidateNumericPrecision now refuses a wrong pair, and + // EffectivePrecision resolves a declared 0 (which means "unspecified") to ACE's 18. + ["decimal-precision-zero"] = new("FixedPoint declaring precision 0 — no digits at all", + db => Numeric(db, precision: 0, scale: 0)), + + ["decimal-precision-over-28"] = new("FixedPoint declaring precision 99; ACE's maximum is 28", + db => Numeric(db, precision: 99, scale: 0)), + + ["decimal-scale-over-precision"] = new("FixedPoint with scale 10 under precision 4 — more decimals than digits", + db => Numeric(db, precision: 4, scale: 10)), + + ["decimal-scale-over-28"] = new("FixedPoint with scale 99", + db => Numeric(db, precision: 18, scale: 99)), + + // ---- Declared length not matching the type's natural width. ValidateFieldWidth enforces only the + // UPPER bound (510 bytes), so a too-NARROW type passes the declaration untouched. + ["int32-one-byte"] = new("Int32 declared 1 byte wide instead of 4", + db => Scalar(db, JetDataType.Int32, 1, fixedLength: true, value: 7)), + + ["double-three-bytes"] = new("Double declared 3 bytes wide instead of 8", + db => Scalar(db, JetDataType.Double, 3, fixedLength: true, value: 1.5d)), + + ["guid-four-bytes"] = new("Guid declared 4 bytes wide instead of 16", + db => Scalar(db, JetDataType.Guid, 4, fixedLength: false, value: Guid.NewGuid())), + + ["currency-two-bytes"] = new("Currency declared 2 bytes wide instead of 8", + db => Scalar(db, JetDataType.Currency, 2, fixedLength: true, value: 1.2345m)), + + ["datetime-four-bytes"] = new("DateTime declared 4 bytes wide instead of 8", + db => Scalar(db, JetDataType.DateTime, 4, fixedLength: true, value: new DateTime(2026, 1, 1))), + + ["int32-over-wide"] = new("Int32 declared 400 bytes wide — inside the 510 cap, far past the type", + db => Scalar(db, JetDataType.Int32, 400, fixedLength: true, value: 7)), + + // ---- Text is UTF-16 on disk, so an odd byte count cannot hold whole characters, and a zero-width + // column cannot hold any. + ["text-odd-byte-length"] = new("Text declared 7 bytes wide — UTF-16 needs an even count", + db => Scalar(db, JetDataType.Text, 7, fixedLength: false, value: "abc")), + + ["text-zero-length"] = new("Text declared 0 bytes wide", + db => Scalar(db, JetDataType.Text, 0, fixedLength: false, value: "")), + + // ---- Storage form contradicting the type. Memo and OLE are long values addressed by a 12-byte + // descriptor; there is no fixed-width form of one. + ["memo-declared-fixed"] = new("Memo declared fixed-length", + db => Scalar(db, JetDataType.Memo, 0, fixedLength: true, value: "text")), + + ["ole-declared-fixed"] = new("OLE declared fixed-length", + db => Scalar(db, JetDataType.Ole, 0, fixedLength: true, value: new byte[] { 1, 2, 3 })), + + // ---- AutoNumber. ACE's COUNTER is Int32 (or a Replication-ID Guid), and a zero increment would make + // every generated value identical. + ["autonumber-on-text"] = new("IsAutoNumber on a Text column", + db => db.CreateTable("S", + [ + new ColumnSpec("Id", JetDataType.Text, 50, IsFixedLength: false, IsAutoNumber: true), + ])), + + ["autonumber-on-double"] = new("IsAutoNumber on a Double column", + db => db.CreateTable("S", + [ + new ColumnSpec("Id", JetDataType.Double, 8, IsFixedLength: true, IsAutoNumber: true), + ])), + + ["autonumber-zero-increment"] = new("AutoNumber with Increment 0 — every row gets the same id", + db => + { + db.CreateTable("S", + [ + new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true, IsAutoNumber: true, + Seed: 1, Increment: 0), + new ColumnSpec("V", JetDataType.Int32, 4, IsFixedLength: true), + ], primaryKey: ["Id"]); + Storage.Table table = db.OpenTable("S"); + table.Insert([null, 1]); + table.Insert([null, 2]); + }), + + // ---- Undocumented descriptor flags Access sets on ITS OWN catalog columns. TdefBuilder's own comment + // says user-table columns leave these clear; nothing stops a caller setting them. + ["system-flags-on-user-column"] = new("SystemFlags 0x10/0x20 (system-catalog, security-id) on a user column", + db => db.CreateTable("S", + [ + new ColumnSpec("Id", JetDataType.Int32, 4, IsFixedLength: true, SystemFlags: 0x10), + new ColumnSpec("Owner", JetDataType.Binary, 100, IsFixedLength: false, SystemFlags: 0x20), + ])), + + // ---- The compression flag is meaningful only on Text and Memo; AccessTypeMapper refuses it elsewhere + // rather than dropping it, and that refusal is SQL-side only. + ["compression-flag-on-int"] = new("SupportsCompressedUnicode on an Int32 column", + db => db.CreateTable("S", + [ + new ColumnSpec("V", JetDataType.Int32, 4, IsFixedLength: true, SupportsCompressedUnicode: true), + ])), + + // ---- A sparse column-id space. Ids are legal individually and the record check uses the high-water, + // so a two-column table numbered 0 and 100 sizes its null bitmap for 101 columns. + ["sparse-column-ids"] = new("Two columns with ids 0 and 100", + db => db.CreateTable("S", + [ + new ColumnSpec("A", JetDataType.Int32, 4, IsFixedLength: true, ColumnId: 0), + new ColumnSpec("B", JetDataType.Int32, 4, IsFixedLength: true, ColumnId: 100), + ])), + + // ---- Index shapes. A text index key is capped at 510 bytes with a truncation checksum; several + // max-width columns together go far past that. + ["index-over-wide-key"] = new("A 5-column index over 255-character text columns — ~2550 bytes of key", + db => + { + ColumnSpec[] columns = [.. Enumerable.Range(0, 5).Select(i => + new ColumnSpec($"C{i}", JetDataType.Text, 510, IsFixedLength: false))]; + db.CreateTable("S", columns); + db.CreateIndex("S", "IX", [.. columns.Select(c => (c.Name, false))]); + db.OpenTable("S").Insert([.. Enumerable.Range(0, 5).Select(object? (_) => new string('x', 255))]); + }), + + ["index-on-memo"] = new("An index over a Memo column holding more than the key can carry", + db => + { + db.CreateTable("S", [new ColumnSpec("M", JetDataType.Memo, 0, IsFixedLength: false)]); + db.CreateIndex("S", "IX", [("M", false)]); + db.OpenTable("S").Insert([new string('m', 20000)]); + }), + + // ---- A version-gated type in a file too old for it. The guard exists (EnsureStorable); this is the + // regression test for the specific hole JetVersion.cs describes. + ["bigint-in-ace12"] = new("Int64 (needs ACE 16) written into an ACE 12 file", + db => db.CreateTable("S", [new ColumnSpec("V", JetDataType.Int64, 8, IsFixedLength: false)])), + + ["datetime2-in-ace12"] = new("DateTimeExtended (needs ACE 17) written into an ACE 12 file", + db => db.CreateTable("S", + [new ColumnSpec("V", JetDataType.DateTimeExtended, 42, IsFixedLength: true)])), + + // ---- Raw row values that do not match the declared column types, straight past the SQL binder. + ["insert-wrong-clr-type"] = new("Table.Insert handing a string to an Int32 column", + db => + { + db.CreateTable("S", [new ColumnSpec("V", JetDataType.Int32, 4, IsFixedLength: true)]); + db.OpenTable("S").Insert(["not a number"]); + }), + + ["insert-too-many-values"] = new("Table.Insert handing three values to a one-column table", + db => + { + db.CreateTable("S", [new ColumnSpec("V", JetDataType.Int32, 4, IsFixedLength: true)]); + db.OpenTable("S").Insert([1, 2, 3]); + }), + + // ---- A calculated column whose expression cannot be evaluated by anyone. + ["calculated-nonsense-expression"] = new("A calculated column over a column that does not exist", + db => db.CreateTable("S", + [ + new ColumnSpec("A", JetDataType.Int32, 4, IsFixedLength: true), + ColumnSpec.Calculated("C", JetDataType.Int32, "[NoSuchColumn] * 2"), + ])), + }; + + /// One column holding the shape under test, plus a row — several only misbehave once a value is + /// laid out against the bad descriptor. + private static void Scalar(JetDatabase db, JetDataType type, int length, bool fixedLength, object? value) + { + db.CreateTable("S", [new ColumnSpec("V", type, length, IsFixedLength: fixedLength)]); + db.OpenTable("S").Insert([value]); + } + + private static void Numeric(JetDatabase db, byte precision, byte scale) + { + db.CreateTable("S", + [ + new ColumnSpec("V", JetDataType.FixedPoint, 17, IsFixedLength: true, Precision: precision, Scale: scale), + ]); + db.OpenTable("S").Insert([1.5m]); + } +} diff --git a/test/LibRed.Engine.AccessTests/AceDecimalPrecisionProbeTest.cs b/test/LibRed.Engine.AccessTests/AceDecimalPrecisionProbeTest.cs new file mode 100644 index 000000000..e139800b6 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/AceDecimalPrecisionProbeTest.cs @@ -0,0 +1,300 @@ +using System.Data.OleDb; +using LibRed; +using LibRed.Data; +using Xunit; + +namespace LibRed.Engine.Tests; + +// A DECIMAL column's declared precision is a contract ACE enforces and LibRed did not. Found by the sweep: a +// workload stored a value wider than the declaration, and ACE then could not read the row. The payload cannot +// enforce it — sign byte plus 128-bit magnitude whatever the column says — and LibRed reading its own file back +// agreed with itself, so only a differential test could find it. +// +// The consequence was never an invalid file, and saying so was an error worth not repeating: ACE's engine reads +// such a row fine via CStr/CDbl/`& ''`. Only the TYPED path fails, because the OLE DB consumer buffer is sized +// from the declared precision — the same shape as the DATETIME2 defect, and the trap rung-3 triage now prevents. +// +// JetTypeCodec.EncodeNumeric now refuses the value, as ACE does; these hold both engines to the same answer. +[Collection(AceCollection.Name)] +public class AceDecimalPrecisionProbeTest(ITestOutputHelper output) +{ + // 20 significant digits, declared into an 18-digit column. + private const decimal TooWide = 12345678901234567890m; + + /// DECIMAL(18,4) permits 14 digits before the point. Both engines must refuse a wider value AND + /// accept the widest that fits — a guard that only says no would pass half of this. + [Theory] + [InlineData("12345678901234567890", false)] + [InlineData("100000000000000", false)] // 10^14: the first magnitude that does not fit + [InlineData("99999999999999.9999", true)] // the declared maximum + [InlineData("1.2345", true)] + public void Libred_accepts_exactly_what_ace_accepts(string literal, bool expectedAccepted) + { + var value = decimal.Parse(literal, System.Globalization.CultureInfo.InvariantCulture); + + bool ace = AceAccepts(value, out string aceDetail); + bool libred = LibRedAccepts(value, out string libredDetail); + output.WriteLine($"{literal,-22} ACE {(ace ? "accepts" : "refuses"),-8} LibRed {(libred ? "accepts" : "refuses")}"); + output.WriteLine($" ACE: {aceDetail}"); + output.WriteLine($" LibRed: {libredDetail}"); + + Assert.Equal(expectedAccepted, ace); // the standard, re-measured rather than assumed + Assert.Equal(ace, libred); // and LibRed agreeing with it + } + + /// Excess scale is coerced, not refused, and both engines must coerce it the SAME way. ACE + /// truncates toward zero; LibRed rounded half-to-even until 2026-09-13, storing 1.2346 for ACE's 1.2345. + /// + /// Undetectable from one side — no error, both files valid, each engine reading its own answer + /// back — and invisible to the sweep, since the file is fine and only the number differs. This test is the + /// only thing between the truncation and a silent regression. The cases separate truncation from the + /// roundings that coincide with it: both midpoints, either side of one, a propagating carry, and the + /// negatives, where truncation parts company with "round down". + [Theory] + [InlineData("1.23455")] + [InlineData("1.23465")] + [InlineData("1.23454")] + [InlineData("1.23456")] + [InlineData("1.99999")] + [InlineData("-1.23456")] + [InlineData("-1.23455")] + public void Libred_coerces_excess_scale_the_way_ace_does(string literal) + { + var value = decimal.Parse(literal, System.Globalization.CultureInfo.InvariantCulture); + + string ace = StoredByAce(value); + string libred = StoredByLibRed(value); + string truncated = (decimal.Truncate(Math.Abs(value) * 10000m) / 10000m * Math.Sign(value)) + .ToString("0.0000", System.Globalization.CultureInfo.InvariantCulture); + + output.WriteLine($"{literal,-10} ACE {ace,-10} LibRed {libred,-10} (truncation gives {truncated})"); + + Assert.Equal(truncated, ace); // the rule, re-measured rather than assumed + Assert.Equal(ace, libred); // and LibRed agreeing with it + } + + private static string StoredByAce(decimal value) + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-scale-"); + try + { + using OleDbConnection connection = AceTestDatabase.Open(path); + Execute(connection, "CREATE TABLE DecScale (Id LONG, V DECIMAL(18,4))"); + Execute(connection, $"INSERT INTO DecScale (Id, V) VALUES (1, {value})"); + using OleDbCommand back = connection.CreateCommand(); + back.CommandText = "SELECT CStr(V) FROM DecScale WHERE Id = 1"; + return Normalise(back.ExecuteScalar()); + } + finally { TemporaryDatabase.Delete(path); } + } + + private static string StoredByLibRed(decimal value) + { + string path = TemporaryDatabase.CreatePath("libred-scale-"); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}"); + using var db = JetDatabase.Open(path, readOnly: false); + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("CREATE TABLE `D` (`Id` LONG, `V` DECIMAL(18,4))"); + engine.ExecuteNonQuery("INSERT INTO `D` (`Id`, `V`) VALUES (1, @v)", + new Dictionary { ["v"] = value }); + return Normalise(engine.ExecuteQuery("SELECT `V` FROM `D`").Rows.First()[0]); + } + finally { TemporaryDatabase.Delete(path); } + } + + /// Rendered at the declared scale, so the comparison is of the NUMBER each engine stored rather + /// than of how each chose to print it. + private static string Normalise(object? value) => + Convert.ToDecimal(value, System.Globalization.CultureInfo.InvariantCulture) + .ToString("0.0000", System.Globalization.CultureInfo.InvariantCulture); + + /// The defect itself, as a regression test: the value must not reach the file. If it ever stores + /// again the row becomes one ACE's typed reader cannot materialise — invisible to LibRed, which reads its + /// own file back without complaint. + [Fact] + public void An_over_precise_decimal_never_reaches_the_file() + { + string path = TemporaryDatabase.CreatePath("libred-decimal-"); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}"); + using (var db = JetDatabase.Open(path, readOnly: false)) + { + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("CREATE TABLE `D` (`Id` LONG, `V` DECIMAL(18,4))"); + + InvalidOperationException refused = Assert.Throws(() => + engine.ExecuteNonQuery("INSERT INTO `D` (`Id`, `V`) VALUES (1, @v)", + new Dictionary { ["v"] = TooWide })); + output.WriteLine(refused.Message); + + Assert.Empty(engine.ExecuteQuery("SELECT `V` FROM `D`").Rows); + } + + // And the file ACE is left with is one it reads through the typed path it would have choked on. + using OleDbConnection connection = AceTestDatabase.Open(path); + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = "SELECT Id, V FROM D"; + using OleDbDataReader reader = command.ExecuteReader(); + while (reader.Read()) { } + } + finally { TemporaryDatabase.Delete(path); } + } + + private static bool AceAccepts(decimal value, out string detail) + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-decimal-"); + try + { + using OleDbConnection connection = AceTestDatabase.Open(path); + Execute(connection, "CREATE TABLE DecProbe (Id LONG, V DECIMAL(18,4))"); + try + { + Execute(connection, $"INSERT INTO DecProbe (Id, V) VALUES (1, {value})"); + using OleDbCommand back = connection.CreateCommand(); + back.CommandText = "SELECT CStr(V) FROM DecProbe WHERE Id = 1"; + detail = $"stored {back.ExecuteScalar()}"; + return true; + } + catch (Exception ex) { detail = ex.Message.Trim(); return false; } + } + finally { TemporaryDatabase.Delete(path); } + } + + private static bool LibRedAccepts(decimal value, out string detail) + { + string path = TemporaryDatabase.CreatePath("libred-decimal-"); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}"); + using var db = JetDatabase.Open(path, readOnly: false); + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("CREATE TABLE `D` (`Id` LONG, `V` DECIMAL(18,4))"); + try + { + engine.ExecuteNonQuery("INSERT INTO `D` (`Id`, `V`) VALUES (1, @v)", + new Dictionary { ["v"] = value }); + detail = $"stored {engine.ExecuteQuery("SELECT `V` FROM `D`").Rows.First()[0]}"; + return true; + } + catch (Exception ex) { detail = $"{ex.GetType().Name}: {ex.Message.Trim()}"; return false; } + } + finally { TemporaryDatabase.Delete(path); } + } + + // -- every write path, not just INSERT ------------------------------------------------------------- + + // 15 integer digits: one more than DECIMAL(18,4)'s 14, and comfortably inside DECIMAL(28,0). + private const string TooWideForUpdate = "123456789012345"; + + /// + /// Whether INSERT is the only gate, or ACE checks every path that puts a value in the column. ALTER is the + /// interesting one: no value is being written at all, the existing rows are what violates the contract, so + /// refusing it makes the declaration an invariant over the whole column rather than a filter. + /// + [Theory] + [InlineData("UPDATE")] + [InlineData("INSERT_SELECT")] + [InlineData("ALTER_NARROW")] + public void Every_write_path_enforces_the_declared_precision(string path) + { + output.WriteLine($"{path}: putting {TooWideForUpdate} (15 integer digits) into a DECIMAL(18,4) — max 14"); + + string aceResult = Describe(() => RunOnAce(path), out bool ace); + string libredResult = Describe(() => RunOnLibRed(path), out bool libred); + output.WriteLine($" ACE: {aceResult}"); + output.WriteLine($" LibRed: {libredResult}"); + + Assert.False(ace, $"ACE was expected to refuse the {path} path but accepted it — {aceResult}"); + Assert.False(libred, $"LibRed accepted the {path} path where ACE refuses it — {libredResult}"); + } + + private static string Describe(Func run, out bool accepted) + { + try + { + string result = run(); + accepted = true; + return result; + } + catch (Exception ex) + { + accepted = false; + return $"REFUSES: {ex.GetType().Name}: {ex.Message.Trim()}"; + } + } + + private static string RunOnAce(string path) + { + string file = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "ace-writepath-"); + try + { + using OleDbConnection connection = AceTestDatabase.Open(file); + foreach (string sql in Script(path)) Execute(connection, sql); + + using OleDbCommand back = connection.CreateCommand(); + back.CommandText = "SELECT CStr(V) FROM W WHERE Id = 1"; + return $"accepts, stores {back.ExecuteScalar()}"; + } + finally { TemporaryDatabase.Delete(file); } + } + + private static string RunOnLibRed(string path) + { + string file = TemporaryDatabase.CreatePath("libred-writepath-"); + File.Delete(file); + try + { + LibRedConnection.CreateDatabase($"Data Source={file}"); + using var db = JetDatabase.Open(file, readOnly: false); + var engine = new QueryEngine(db); + foreach (string sql in Script(path)) engine.ExecuteNonQuery(sql); + return $"accepts, stores {engine.ExecuteQuery("SELECT `V` FROM `W` WHERE `Id` = 1").Rows.First()[0]}"; + } + finally { TemporaryDatabase.Delete(file); } + } + + /// The same statements for both engines, so the only variable is the engine. Each script ends with + /// the too-wide value in a DECIMAL(18,4) column. + private static string[] Script(string path) => path switch + { + // A row that fits, then an UPDATE that does not. + "UPDATE" => + [ + "CREATE TABLE W (Id LONG, V DECIMAL(18,4))", + "INSERT INTO W (Id, V) VALUES (1, 1)", + $"UPDATE W SET V = {TooWideForUpdate} WHERE Id = 1", + ], + // The value arrives from a query over a column wide enough to hold it. + "INSERT_SELECT" => + [ + "CREATE TABLE W (Id LONG, V DECIMAL(18,4))", + "CREATE TABLE Src (Id LONG, V DECIMAL(28,0))", + $"INSERT INTO Src (Id, V) VALUES (1, {TooWideForUpdate})", + "INSERT INTO W (Id, V) SELECT Id, V FROM Src", + ], + // The column narrows underneath a value that was legal when written. + "ALTER_NARROW" => + [ + "CREATE TABLE W (Id LONG, V DECIMAL(28,0))", + $"INSERT INTO W (Id, V) VALUES (1, {TooWideForUpdate})", + "ALTER TABLE W ALTER COLUMN V DECIMAL(18,4)", + ], + _ => throw new ArgumentOutOfRangeException(nameof(path)), + }; + + private static void Execute(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} diff --git a/test/LibRed.Engine.AccessTests/AceValidityLadder.cs b/test/LibRed.Engine.AccessTests/AceValidityLadder.cs new file mode 100644 index 000000000..315b76969 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/AceValidityLadder.cs @@ -0,0 +1,186 @@ +using System.Data; +using System.Data.OleDb; +using LibRed; +using LibRed.Catalog; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The oracle both write-validity arms share: given a file LibRed wrote, does Access's own engine accept it? +/// Five rungs, each a gate: +/// +/// the provider connects at all (a refusal here is "Unrecognized database format") +/// Tables / Columns / Indexes / Foreign_Keys enumerate +/// SELECT * over every user table, materialising every value (row decode + long-value chase) +/// ACE inserts, updates and deletes through its own allocator and index maintenance +/// LibRed reads the file back after ACE has written to it +/// +/// Rung 4 earns its place: ACE writing means ACE trusting page 1's free map enough to allocate against it, and +/// a file can pass 1-3 on a free map that is quietly wrong. +/// +/// Shared so both arms judge by one standard — a finding from either has to be comparable. +internal static class AceValidityLadder +{ + /// is null when ACE accepted the file. + /// Tables whose TYPED read failed while the engine still returned the rows as text + /// — an ACE provider defect, not a bad file. See the triage note in . + public sealed record Verdict(string? Finding, List TypedOnly); + + /// An unconstrained table for rung 4, or null to skip it. It must be one ACE + /// cannot refuse for a reason about the data rather than the file. + public static Verdict Check(string path, string? writeTarget) + { + var typedOnly = new List(); + + OleDbConnection connection; + try + { + // Three attempts, not the helper's twelve: a file ACE genuinely rejects would otherwise burn 24 + // provider opens per case, and ACE's transient-open flakiness needs far less than that. + connection = AceTestDatabase.Open(path, attempts: 3); + } + catch (Exception ex) + { + return new Verdict($"rung 1 (ACE cannot open the file): {Flatten(ex.Message)}", typedOnly); + } + + using (connection) + { + var tables = new List(); + try + { + foreach ((Guid guid, string name) in new[] + { + (OleDbSchemaGuid.Tables, "Tables"), + (OleDbSchemaGuid.Columns, "Columns"), + (OleDbSchemaGuid.Indexes, "Indexes"), + (OleDbSchemaGuid.Foreign_Keys, "Foreign_Keys"), + }) + { + using DataTable schema = connection.GetOleDbSchemaTable(guid, null) + ?? throw new InvalidOperationException($"{name} returned no schema table."); + + if (guid != OleDbSchemaGuid.Tables) continue; + foreach (DataRow row in schema.Rows) + if ((string)row["TABLE_TYPE"] == "TABLE") + tables.Add((string)row["TABLE_NAME"]); + } + } + catch (Exception ex) + { + return new Verdict( + $"rung 2 (ACE cannot enumerate the schema): {ex.GetType().Name}: {Flatten(ex.Message)}", + typedOnly); + } + + foreach (string table in tables) + { + try + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = $"SELECT * FROM [{table}]"; + using OleDbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + for (int i = 0; i < reader.FieldCount; i++) + _ = reader.GetValue(i); // materialise it: this is what chases a long value + } + catch (Exception ex) + { + // A typed read failing is NOT evidence the file is bad — assuming otherwise cost this work + // two false findings. ACE's OLE DB provider breaks on values its own engine handles: + // DATETIME2 outright (docs/format/data-types.md's footnote) and a decimal wider than its + // declared precision, where the consumer buffer is sized from the declaration. + // + // So ask the ENGINE for the same rows as text first. `& ''` not CStr(), which raises + // "Invalid use of Null". If that works, only the typed marshalling is broken. + if (ReadsAsText(connection, table)) + { + typedOnly.Add( + $"ACE's TYPED read of [{table}] fails though the engine returns the same rows as " + + $"text: {ex.GetType().Name}: {Flatten(ex.Message)}"); + continue; + } + + return new Verdict( + $"rung 3 (ACE cannot read [{table}]): {ex.GetType().Name}: {Flatten(ex.Message)}", + typedOnly); + } + } + + if (writeTarget is not null) + { + try + { + Execute(connection, $"INSERT INTO [{writeTarget}] ([K], [V]) VALUES (9001, 'ace')"); + Execute(connection, $"UPDATE [{writeTarget}] SET [V] = 'ace2' WHERE [K] = 9001"); + Execute(connection, $"DELETE FROM [{writeTarget}] WHERE [K] = 9001"); + } + catch (Exception ex) + { + return new Verdict($"rung 4 (ACE cannot write): {ex.GetType().Name}: {Flatten(ex.Message)}", + typedOnly); + } + } + } + + return new Verdict(LibRedReopen(path), typedOnly); + } + + /// Every row of with each column converted to text, so no typed buffer is + /// allocated. True means the engine can read the rows and any typed-path failure is the provider's. + private static bool ReadsAsText(OleDbConnection connection, string table) + { + try + { + var columns = new List(); + using (DataTable? schema = connection.GetOleDbSchemaTable( + OleDbSchemaGuid.Columns, [null, null, table, null])) + { + if (schema is null) return false; + foreach (DataRow row in schema.Rows) columns.Add((string)row["COLUMN_NAME"]); + } + if (columns.Count == 0) return false; + + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = + $"SELECT {string.Join(", ", columns.Select(c => $"[{c}] & ''"))} FROM [{table}]"; + using OleDbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + for (int i = 0; i < reader.FieldCount; i++) + _ = reader.GetValue(i); + return true; + } + catch (Exception) + { + return false; // the engine cannot produce the rows either — the file really is the problem + } + } + + /// Rung 5: LibRed reads back a file ACE has just written to. + private static string? LibRedReopen(string path) + { + try + { + using var db = JetDatabase.Open(path, readOnly: true); + foreach (TableDef table in db.Catalog.UserTables) + foreach (object?[] row in db.OpenTable(table.Name).Rows()) + _ = row.Length; + return null; + } + catch (Exception ex) + { + return $"rung 5 (LibRed cannot reread what ACE wrote): {ex.GetType().Name}: {Flatten(ex.Message)}"; + } + } + + private static void Execute(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + public static string Flatten(string message) => + string.Join(' ', message.Split('\n', '\r').Select(l => l.Trim()).Where(l => l.Length > 0)); +} diff --git a/test/LibRed.Engine.AccessTests/AceWriteValiditySweepProbeTests.cs b/test/LibRed.Engine.AccessTests/AceWriteValiditySweepProbeTests.cs new file mode 100644 index 000000000..b57fc0b8c --- /dev/null +++ b/test/LibRed.Engine.AccessTests/AceWriteValiditySweepProbeTests.cs @@ -0,0 +1,635 @@ +using System.Data; +using System.Data.OleDb; +using System.Text; +using LibRed; +using LibRed.Catalog; +using LibRed.Data; +using LibRed.Formats; +using Xunit; + +namespace LibRed.Engine.Tests; + +// The OUTBOUND direction of CorruptFileSweepProbeTests: that one feeds damaged bytes INTO LibRed's reader; this +// runs LibRed's WRITER over randomised workloads and asks whether ACE refuses the result. The oracle is +// AceValidityLadder. +// +// A sweep rather than more hand-written parity tests because the ~130 ACE cross-checks each pin a shape someone +// was already looking at. What they miss is the INTERACTION — a column dropped after an index was built over it, +// a long value relocated into space a deleted row freed, a format raised mid-rebuild. +// +// DAO Compact & Repair is deliberately not a rung: a stricter bar than "ACE opens and round-trips", which the +// repo has not claimed (README lists it as open), so it would bury real finds under known noise. +// +// Committed small and fixed-seed — a regression guard, not a search. Discovery means: +// +// $env:LIBRED_ACE_SWEEP = 400 ; $env:LIBRED_ACE_SWEEP_SEED = 7 +// +[Collection(AceCollection.Name)] +public class AceWriteValiditySweepProbeTests(ITestOutputHelper output) : TempDatabaseTest +{ + /// Workloads run when nothing overrides it. Small on purpose — see the note above. + private const int CommittedWorkloads = 8; + + [Fact] + public void No_generated_workload_produces_a_file_ace_calls_invalid() + { + int workloads = ReadInt("LIBRED_ACE_SWEEP", CommittedWorkloads); + int seed = ReadInt("LIBRED_ACE_SWEEP_SEED", 20260912); + + // Asked once: a BIGINT workload on an ACE 12 engine would fail for a reason unrelated to the file. + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + bool bigInt = AceTestDatabase.SupportsColumnType(northwind, "BIGINT"); + JetVersion newest = NewestVersionAceOpens(northwind); + output.WriteLine($"seed {seed}, {workloads} workloads; ACE supports BIGINT={bigInt}, opens up to {newest}"); + + // FINDING = a file ACE will not accept, which is what this exists for. LEAK = a statement that threw + // outside the refusal contract; a real defect but a caller-facing one, so the workload carries on and + // the ladder still answers whether it left a readable file. + var findings = new List(); + var leaks = new List(); + for (int i = 0; i < workloads; i++) + { + int workloadSeed = seed + i; + Plan plan = Plan.Generate(new Random(workloadSeed), bigInt, newest); + + Outcome outcome = RunAndValidate(plan, plan.Steps.Count); + leaks.AddRange(outcome.Leaks); + + if (outcome.AceVerdict is not { } failure) continue; + + // Replay prefix by prefix to name the culprit. The plan comes purely from the seed and never + // branches on whether a statement succeeded, so a replay is the same sequence. + string located = Locate(plan); + findings.Add($"seed {workloadSeed} ({plan.Describe()}): {failure}\n{located}"); + output.WriteLine(findings[^1]); + } + + string[] grouped = leaks + .GroupBy(l => l.Signature) + .Select(g => $" x{g.Count(),-5} {g.Key}\n e.g. {g.First().Example}") + .ToArray(); + foreach (string leak in grouped) output.WriteLine(leak); + + Assert.True(findings.Count == 0 && leaks.Count == 0, + $"{findings.Count} of {workloads} workloads produced a file ACE would not accept:\n\n" + + string.Join("\n\n", findings) + + $"\n\n{leaks.Count} statement(s) in {grouped.Length} shape(s) failed outside the refusal " + + $"contract:\n\n" + string.Join("\n", grouped)); + } + + /// The newest format the installed ACE opens. An older engine refuses a file whose version byte is + /// past it ("requires a newer version of Microsoft Access") — CI's ACE 2016 redistributable opens neither + /// 0x05 nor 0x06. Asked through the type each format was introduced for, the same probe the rest of the + /// suite gates on. + private static JetVersion NewestVersionAceOpens(string northwind) => + AceTestDatabase.SupportsColumnType(northwind, "DATETIME2") ? JetVersion.Version17_2019 + : AceTestDatabase.SupportsColumnType(northwind, "BIGINT") ? JetVersion.Version16_2016 + : JetVersion.Version14_2010; + + /// A workload's two verdicts — see the note in the test body. + private sealed record Outcome(string? AceVerdict, List Leaks); + + /// A statement that failed outside the refusal contract. Grouped by signature when reported: one + /// root cause routinely accounts for scores of statements. + private sealed record Leak(string Signature, string Example); + + /// The floor the sweep stands on: an EMPTY database at each format, opened by ACE. If this fails + /// for a version, every workload at that version fails too and says nothing about its statements — which is + /// how the 0x04 defect first surfaced. + [Theory] + [InlineData(JetVersion.Version4, ".mdb")] + [InlineData(JetVersion.Version12_2007, ".accdb")] + [InlineData(JetVersion.Version14_2010, ".accdb")] + [InlineData(JetVersion.Version16_2016, ".accdb")] + [InlineData(JetVersion.Version17_2019, ".accdb")] + public void Ace_opens_an_empty_database_at_every_format_libred_creates(JetVersion version, string extension) + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + Assert.SkipWhen(version > NewestVersionAceOpens(northwind), + $"The installed ACE predates {version} and refuses the file as needing a newer Access, which says " + + "nothing about LibRed."); + + string path = TemporaryDatabase.CreatePath("libred-empty-", extension); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}", collation: null, version: version); + + using var stream = File.OpenRead(path); + stream.Seek(0x14, SeekOrigin.Begin); + output.WriteLine($"{version}: version byte 0x{stream.ReadByte():X2}"); + stream.Dispose(); + + using OleDbConnection connection = AceTestDatabase.Open(path, attempts: 3); + using DataTable? tables = connection.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null); + Assert.NotNull(tables); + } + finally { TemporaryDatabase.Delete(path); } + } + + /// Creating an ACE 15 database is refused outright — see the evidence in + /// . + [Fact] + public void Libred_refuses_to_create_an_ace_15_database() + { + string path = TemporaryDatabase.CreatePath("libred-ace15-create-", ".accdb"); + File.Delete(path); + try + { + NotSupportedException refused = Assert.Throws(() => + LibRedConnection.CreateDatabase($"Data Source={path}", collation: null, + version: JetVersion.Version15_2013)); + + output.WriteLine(refused.Message); + Assert.Contains("Version14_2010", refused.Message); // the caller is told what to ask for instead + Assert.False(File.Exists(path), "a refused creation must not leave a file behind"); + } + finally { TemporaryDatabase.Delete(path); } + } + + /// The measurement the creation guard rests on: ACE does not merely avoid the 0x04 version byte, + /// it REFUSES a file carrying one, and restamping 0x14 to 0x03 makes the identical bytes open. The spec + /// previously inferred "reserved" from absence; this is the stronger fact. + /// Built at 2010 and stamped by hand, since DatabaseCreator now refuses to write 0x04 — the + /// evidence must not depend on the guard being absent. + [Fact] + public void Ace_refuses_the_0x04_version_byte_and_nothing_else_about_the_file() + { + string path = TemporaryDatabase.CreatePath("libred-ace15-", ".accdb"); + File.Delete(path); + try + { + LibRedConnection.CreateDatabase($"Data Source={path}", collation: null, + version: JetVersion.Version14_2010); + + // Sanity: as created, at 0x03, ACE opens it. Without this the refusal below would prove nothing — + // a file ACE rejects for some unrelated reason would look identical. + using (OleDbConnection asBuilt = AceTestDatabase.Open(path, attempts: 3)) + Assert.NotNull(asBuilt.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null)); + + Stamp(path, 0x04); + Assert.Throws(() => AceTestDatabase.Open(path, attempts: 2).Dispose()); + + Stamp(path, 0x03); + using OleDbConnection restored = AceTestDatabase.Open(path, attempts: 3); + Assert.NotNull(restored.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null)); + } + finally { TemporaryDatabase.Delete(path); } + } + + private static void Stamp(string path, byte version) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + stream.Seek(0x14, SeekOrigin.Begin); + stream.WriteByte(version); + } + + /// + /// Builds the database, applies the first statements, then walks the ACE ladder. + /// + private Outcome RunAndValidate(Plan plan, int steps) + { + var leaks = new List(); + string path = TemporaryDatabase.CreatePath("libred-ace-sweep-", plan.Extension); + File.Delete(path); // CreateDatabase synthesises the file and refuses an existing one + try + { + LibRedConnection.CreateDatabase($"Data Source={path}", plan.Collation, plan.Version); + + using (var db = JetDatabase.Open(path, readOnly: false)) + { + var engine = new QueryEngine(db); + for (int i = 0; i < steps; i++) + { + Step step = plan.Steps[i]; + try + { + step.Run(db, engine); + } + catch (Exception ex) when (IsRefusal(ex)) + { + // Allowed: a duplicate key, a column an earlier step dropped, an unimplemented feature. + // What matters is that the refusal leaves a file ACE accepts — the ladder's job. + _ = ex; + } + catch (Exception ex) + { + // Outside the contract. Recorded, and the workload CARRIES ON — statements are atomic, + // so the question is whether this left a readable file, and stopping would never ask it. + leaks.Add(new Leak($"{ex.GetType().Name}: {Flatten(ex.Message)}", step.Label)); + } + } + } + + AceValidityLadder.Verdict verdict = AceValidityLadder.Check(path, Plan.AceWriteTarget); + leaks.AddRange(verdict.TypedOnly.Select(t => new Leak(t, plan.Describe()))); + return new Outcome(verdict.Finding, leaks); + } + catch (Exception ex) + { + return new Outcome($"rung 0 (LibRed itself): {ex.GetType().Name}: {Flatten(ex.Message)}", leaks); + } + finally { TemporaryDatabase.Delete(path); } + } + + + /// Replays the plan prefix by prefix to name the first statement ACE will not survive. + private string Locate(Plan plan) + { + for (int steps = 1; steps <= plan.Steps.Count; steps++) + { + if (RunAndValidate(plan, steps).AceVerdict is not { } failure) continue; + + var report = new StringBuilder($" first bad prefix is {steps} step(s):\n"); + for (int i = 0; i < steps; i++) + report.Append($" {(i == steps - 1 ? "->" : " ")} {plan.Steps[i].Label}\n"); + return report.Append($" {failure}").ToString(); + } + + // The whole plan failed but no prefix of it does. That is a real result, not a harness bug: it means the + // damage needs the complete sequence, so the log below IS the reproduction. + return " no shorter prefix reproduces it; the full plan is:\n" + + string.Join("\n", plan.Steps.Select(s => $" {s.Label}")); + } + + private static void Execute(OleDbConnection connection, string sql) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + /// The exception types a generated statement may legitimately raise. Everything else — a null + /// reference, an index out of range, an InvalidDataException from LibRed's reader over its own writes — is + /// a defect and escapes to be reported. + private static bool IsRefusal(Exception ex) => + ex is NotSupportedException // a feature LibRed does not implement yet + or InvalidOperationException // a constraint, a missing object, a bad declared size + or ArgumentException + or FormatException + or OverflowException; + + private static string Flatten(string message) => + string.Join(' ', message.Split('\n', '\r').Select(l => l.Trim()).Where(l => l.Length > 0)); + + private static int ReadInt(string variable, int fallback) => + int.TryParse(Environment.GetEnvironmentVariable(variable), out int value) && value > 0 ? value : fallback; + + // --------------------------------------------------------------------------------------------------- + // The generator + // --------------------------------------------------------------------------------------------------- + + private sealed record Step(string Label, Action Run); + + /// A workload, fixed at generation time. It never branches on execution, so a seed always yields + /// the same sequence — which is what lets bisect a failure. + private sealed class Plan + { + /// The unconstrained table rung 4 writes into. Every plan creates it first. + public const string AceWriteTarget = "AceTarget"; + + public required JetVersion Version { get; init; } + public required Collation Collation { get; init; } + public required List Steps { get; init; } + + /// Jet 4 is an .mdb; everything from ACE 12 on is an .accdb. + public string Extension => Version <= JetVersion.Version4 ? ".mdb" : ".accdb"; + + public string Describe() => + $"{Version}, collation v{Collation.Version}, {Steps.Count} steps"; + + public static Plan Generate(Random random, bool bigInt, JetVersion newest) + { + // Jet 4 through ACE 17. Version3 LibRed does not create at all; Version15_2013 it now refuses, + // because ACE will not open a 0x04 file — see Libred_refuses_to_create_an_ace_15_database. + // A version newer than the installed ACE is clamped rather than dropped from the list, so a seed + // consumes the same random numbers — and so describes the same statements — on every engine. + JetVersion version = Pick(random, + [ + JetVersion.Version4, JetVersion.Version12_2007, JetVersion.Version14_2010, + JetVersion.Version16_2016, JetVersion.Version17_2019, + ]); + if (version > newest) version = newest; + + var generator = new Generator(random, version, bigInt); + return new Plan + { + Version = version, + Collation = random.Next(2) == 0 ? Collation.GeneralLegacy : Collation.General, + Steps = generator.Build(8 + random.Next(24)), + }; + } + + private static T Pick(Random random, IReadOnlyList options) => options[random.Next(options.Count)]; + } + + /// Builds statements against a MODEL of the schema, not the live database, so most of them are + /// meaningful. The model drifts whenever a statement is refused, and that drift is welcome — it produces + /// sequences a fixture never would. + private sealed class Generator(Random random, JetVersion version, bool bigInt) + { + private readonly List _tables = []; + private readonly List _steps = []; + private int _names; + + private sealed record Column(string Name, string Sql, JetDataType Kind, bool Indexable); + private sealed record TableModel(string Name, List Columns, List Indexes); + + public List Build(int operations) + { + // Rung 4's target, kept OUT of the model so no generated statement can drop it — otherwise an "ACE + // cannot write" verdict would usually mean the harness had removed the table. + Sql($"CREATE TABLE [{Plan.AceWriteTarget}] ([K] LONG, [V] TEXT(50))"); + + CreateTable(); // one real table before the random walk, so early operations have something to hit + + for (int i = 0; i < operations; i++) + { + switch (random.Next(14)) + { + case 0: case 1: CreateTable(); break; + case 2: case 3: case 4: case 5: Insert(); break; + case 6: Update(); break; + case 7: Delete(); break; + case 8: CreateIndex(); break; + case 9: AddColumn(); break; + case 10: DropColumn(); break; + case 11: AlterColumn(); break; + case 12: DropObject(); break; + case 13: CreateView(); break; + } + } + + return _steps; + } + + // -- operations --------------------------------------------------------------------------------- + + private void CreateTable() + { + string name = $"T{++_names}"; + // An AutoNumber PK on every table: gives FKs something to point at, and puts the high-water + // machinery under every workload rather than a chosen few. + var columns = new List { new("Id", "COUNTER", JetDataType.Int32, true) }; + int width = 1 + random.Next(8); + for (int i = 0; i < width; i++) columns.Add(NewColumn($"C{i}")); + + string body = string.Join(", ", columns.Select(c => + c.Name == "Id" ? "[Id] COUNTER PRIMARY KEY" : $"[{c.Name}] {c.Sql}{(random.Next(6) == 0 ? " NOT NULL" : "")}")); + + Sql($"CREATE TABLE [{name}] ({body})"); + _tables.Add(new TableModel(name, columns, [])); + + // Sometimes an FK back to an earlier table — the step that puts a relationship's logical-index + // linkage on two TDEFs at once, a shape ACE is fussy about. + if (_tables.Count > 1 && random.Next(3) == 0) + { + TableModel parent = _tables[random.Next(_tables.Count - 1)]; + if (parent.Columns.Any(c => c.Name == "Id")) + { + Sql($"ALTER TABLE [{name}] ADD COLUMN [Ref] LONG"); + columns.Add(new Column("Ref", "LONG", JetDataType.Int32, true)); + string action = Pick(["", " ON DELETE CASCADE", " ON UPDATE CASCADE", " ON DELETE SET NULL"]); + Sql($"ALTER TABLE [{name}] ADD CONSTRAINT [FK{_names}] " + + $"FOREIGN KEY ([Ref]) REFERENCES [{parent.Name}] ([Id]){action}"); + } + } + } + + private void Insert() + { + if (Table() is not { } table) return; + Column[] columns = table.Columns.Where(c => c.Sql != "COUNTER").ToArray(); + if (columns.Length == 0) return; + + int rows = Pick([1, 1, 1, 3, 20, 120]); // 120 crosses the 255-rows-per-page ceiling when repeated + for (int r = 0; r < rows; r++) + { + var parameters = new Dictionary(); + for (int i = 0; i < columns.Length; i++) parameters[$"p{i}"] = Value(columns[i].Kind); + + Sql($"INSERT INTO [{table.Name}] ({string.Join(", ", columns.Select(c => $"[{c.Name}]"))}) " + + $"VALUES ({string.Join(", ", columns.Select((_, i) => $"@p{i}"))})", + parameters); + } + } + + private void Update() + { + if (Table() is not { } table) return; + Column[] columns = table.Columns.Where(c => c.Sql != "COUNTER").ToArray(); + if (columns.Length == 0) return; + + Column target = Pick(columns); + // Changing a long value's length forces the row to relocate — where a stale index entry or an + // unreclaimed LVAL page shows up. + Sql($"UPDATE [{table.Name}] SET [{target.Name}] = @v", + new Dictionary { ["v"] = Value(target.Kind) }); + } + + private void Delete() + { + if (Table() is not { } table) return; + // Partial far more often than total — a half-emptied page is the interesting one, and deletion is + // where writer parity breaks: dead bytes have no reader to catch a wrong marker. + Sql(random.Next(5) == 0 + ? $"DELETE FROM [{table.Name}]" + : $"DELETE FROM [{table.Name}] WHERE [Id] > {random.Next(50)}"); + } + + private void CreateIndex() + { + if (Table() is not { } table) return; + Column[] indexable = table.Columns.Where(c => c.Indexable).ToArray(); + if (indexable.Length == 0) return; + + int columns = 1 + random.Next(Math.Min(3, indexable.Length)); + string[] chosen = indexable.OrderBy(_ => random.Next()).Take(columns) + .Select(c => $"[{c.Name}]{(random.Next(2) == 0 ? " DESC" : "")}").ToArray(); + + string name = $"IX{table.Name}_{table.Indexes.Count}"; + string unique = random.Next(4) == 0 ? "UNIQUE " : ""; + Sql($"CREATE {unique}INDEX [{name}] ON [{table.Name}] ({string.Join(", ", chosen)})"); + table.Indexes.Add(name); + } + + private void AddColumn() + { + if (Table() is not { } table) return; + Column column = NewColumn($"A{table.Columns.Count}"); + Sql($"ALTER TABLE [{table.Name}] ADD COLUMN [{column.Name}] {column.Sql}"); + table.Columns.Add(column); + } + + private void DropColumn() + { + if (Table() is not { } table) return; + Column[] droppable = table.Columns.Where(c => c.Name != "Id").ToArray(); + if (droppable.Length == 0) return; + + Column column = Pick(droppable); + Sql($"ALTER TABLE [{table.Name}] DROP COLUMN [{column.Name}]"); + table.Columns.Remove(column); + } + + private void AlterColumn() + { + if (Table() is not { } table) return; + Column[] alterable = table.Columns.Where(c => c.Name != "Id").ToArray(); + if (alterable.Length == 0) return; + + Column column = Pick(alterable); + Column replacement = NewColumn(column.Name); + Sql($"ALTER TABLE [{table.Name}] ALTER COLUMN [{column.Name}] {replacement.Sql}"); + table.Columns[table.Columns.IndexOf(column)] = replacement; + } + + private void DropObject() + { + if (Table() is not { } table) return; + + if (table.Indexes.Count > 0 && random.Next(2) == 0) + { + string index = Pick(table.Indexes); + Sql($"DROP INDEX [{index}] ON [{table.Name}]"); + table.Indexes.Remove(index); + return; + } + + Sql($"DROP TABLE [{table.Name}]"); + _tables.Remove(table); + } + + private void CreateView() + { + if (Table() is not { } table) return; + Sql($"CREATE VIEW [V{++_names}] AS SELECT * FROM [{table.Name}]"); + } + + // -- column and value menus --------------------------------------------------------------------- + + private Column NewColumn(string name) + { + (string sql, JetDataType kind, bool indexable)[] menu = + [ + ("LONG", JetDataType.Int32, true), + ("SMALLINT", JetDataType.Int16, true), + ("BYTE", JetDataType.Byte, true), + ("REAL", JetDataType.Single, true), + ("FLOAT", JetDataType.Double, true), + ("CURRENCY", JetDataType.Currency, true), + ("DATETIME", JetDataType.DateTime, true), + ("BIT", JetDataType.Boolean, true), + ("GUID", JetDataType.Guid, true), + ("DECIMAL(18,4)", JetDataType.FixedPoint, true), + ("DECIMAL(28,0)", JetDataType.FixedPoint, true), + // Both ends of the text width range: 1 character, and the 255 ACE caps a char column at. The + // wide one is also what pushes an index key past the 510-byte entry limit into truncation. + ("TEXT(1)", JetDataType.Text, true), + ("TEXT(50)", JetDataType.Text, true), + ("TEXT(255)", JetDataType.Text, true), + ("CHAR(10)", JetDataType.Text, true), + ("VARCHAR(255) WITH COMPRESSION", JetDataType.Text, true), + ("BINARY(16)", JetDataType.Binary, true), + ("VARBINARY(510)", JetDataType.Binary, true), + ("MEMO", JetDataType.Memo, true), + ("OLEOBJECT", JetDataType.Ole, false), + ]; + + var options = menu.ToList(); + // BIGINT only where both the format and the installed engine can hold it; its presence is what + // raises the version byte mid-sequence. The provider reads it correctly, so rung 3 can judge it. + if (bigInt && version >= JetVersion.Version16_2016) options.Add(("BIGINT", JetDataType.Int64, true)); + + // DATETIME2 is ABSENT deliberately: ACE's OLE DB provider cannot read the type back (throws on some + // values, returns the WRONG MONTH on others — see the "Reading DATETIME2 through ACE's own drivers" + // footnote in docs/format/data-types.md), so rung 3 reports the reader's defect whatever LibRed + // wrote, and an early sweep duly "found" one. Covered by DateTime2CreatedDatabaseAccessTests, which + // reads through scalar functions instead. + + (string sql, JetDataType kind, bool indexable) chosen = Pick(options); + return new Column(name, chosen.sql, chosen.kind, chosen.indexable); + } + + /// Nulls and extremes are deliberately common: the sign transforms in index-key encoding and + /// the boundaries in long-value storage are where a writer disagrees with ACE, and middle values agree + /// with almost any encoding. + private object? Value(JetDataType kind) + { + if (random.Next(8) == 0) return null; + + return kind switch + { + JetDataType.Boolean => random.Next(2) == 0, + JetDataType.Byte => (byte)Pick([0, 1, 127, 255]), + JetDataType.Int16 => (short)Pick([0, -1, 1, short.MaxValue, short.MinValue]), + JetDataType.Int32 => Pick([0, -1, 1, int.MaxValue, int.MinValue, random.Next()]), + JetDataType.Int64 => Pick([0L, -1L, 1L, long.MaxValue, long.MinValue]), + JetDataType.Single => Pick([0f, -1f, 1f, float.MaxValue, float.MinValue, float.Epsilon]), + JetDataType.Double => Pick([0d, -1d, 1d, double.MaxValue, double.MinValue, double.Epsilon]), + // Currency is OLE Automation's CY: an int64 scaled by 10,000, so exactly four decimal places. + JetDataType.Currency => Pick([0m, -1.2345m, 922337203685477.5807m, -922337203685477.5808m]), + JetDataType.FixedPoint => Pick([0m, -1m, 1.2345m, 79228162514264337593543950335m / 1000000000m]), + // Below the epoch the OA time fraction stays positive, which is where date ordering diverges. + JetDataType.DateTime or JetDataType.DateTimeExtended => Pick( + [ + new DateTime(1899, 12, 30), new DateTime(1800, 6, 15, 13, 45, 30), new DateTime(1, 1, 1), + new DateTime(9999, 12, 31, 23, 59, 59), DateTime.UnixEpoch, new DateTime(2026, 9, 12), + ]), + JetDataType.Guid => Guid.NewGuid(), + JetDataType.Binary or JetDataType.Ole => Bytes(), + _ => Text(), + }; + } + + /// Text that crosses the awkward boundaries: empty, the compressed-Unicode common case, a + /// string that cannot be compressed, astral characters, and lengths that push a long value from inline + /// to a single LVAL page to a chain. + private string Text() => Pick>( + [ + () => "", + () => "a", + () => new string('x', 255), + () => "O'Brien \"quoted\" [bracketed] `backticked`", + () => "naïve café Ω日本語", // non-Latin1: defeats compressed-Unicode encoding + () => "\U0001F600\U00020000", // above the BMP — surrogate pairs in an index key + () => "ȩ́", // combining marks: one grapheme, three code units + () => new string('m', Pick([1, 40, 2000, 4000, 8000, 60000])), + ])(); + + private byte[] Bytes() + { + int length = Pick([0, 1, 16, 255, 510, 4000, 20000]); + var bytes = new byte[length]; + random.NextBytes(bytes); + return bytes; + } + + // -- plumbing ----------------------------------------------------------------------------------- + + private TableModel? Table() => + _tables.Count == 0 ? null : _tables[random.Next(_tables.Count)]; + + private T Pick(IReadOnlyList options) => options[random.Next(options.Count)]; + + private void Sql(string sql, Dictionary? parameters = null) + => _steps.Add(new Step(Summarise(sql, parameters), + (_, engine) => engine.ExecuteNonQuery(sql, parameters))); + + /// A one-line label for the failure log — parameter values folded in, long text elided. + private static string Summarise(string sql, Dictionary? parameters) + { + if (parameters is null or { Count: 0 }) return sql; + string rendered = string.Join(", ", parameters.Select(p => $"{p.Key}={Render(p.Value)}")); + return $"{sql} -- {rendered}"; + } + + private static string Render(object? value) => value switch + { + null => "NULL", + string s => s.Length > 24 ? $"'{s[..12]}…' ({s.Length} chars)" : $"'{s}'", + byte[] b => $"{b.Length} bytes", + _ => value.ToString() ?? "", + }; + } +} diff --git a/test/LibRed.Engine.AccessTests/DateTime2LocaleAccessTests.cs b/test/LibRed.Engine.AccessTests/DateTime2LocaleAccessTests.cs index 147943277..6b7d0d6d8 100644 --- a/test/LibRed.Engine.AccessTests/DateTime2LocaleAccessTests.cs +++ b/test/LibRed.Engine.AccessTests/DateTime2LocaleAccessTests.cs @@ -37,10 +37,8 @@ public class DateTime2LocaleAccessTests(ITestOutputHelper output) : TempDatabase [MemberData(nameof(Locales))] public void Ace_gives_date_time_extended_only_the_primary_language_id(string locale, int langId) { - object? engine = DaoEngine(); - Assert.SkipWhen(engine is null, "DAO is unavailable in this process."); - - string path = Recollated(engine!, locale); + object engine = DateTime2AndDao(); + string path = Recollated(engine, locale); try { CreateThroughAce(path); @@ -67,11 +65,9 @@ public void Ace_gives_date_time_extended_only_the_primary_language_id(string loc [InlineData(";LANGID=0x041D;CP=1252;COUNTRY=0")] public void Libred_writes_the_same_descriptor_as_ace(string locale) { - object? engine = DaoEngine(); - Assert.SkipWhen(engine is null, "DAO is unavailable in this process."); - - string ace = Recollated(engine!, locale); - string libred = Recollated(engine!, locale); + object engine = DateTime2AndDao(); + string ace = Recollated(engine, locale); + string libred = Recollated(engine, locale); try { CreateThroughAce(ace); @@ -147,14 +143,16 @@ private static Dictionary Descriptors(string path, string table) return result; } - private static object? DaoEngine() + /// A DAO engine, once this ACE is known to take DATETIME2. An ACE below 17 cannot create the column + /// at all — CI installs the 2016 redistributable — so that is asked first, rather than after a DAO compaction + /// that could only end in a skip. + private static object DateTime2AndDao() { - foreach (int n in new[] { 170, 160, 150, 140, 130, 120 }) - { - Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{n}"); - if (type is null) continue; - try { return Activator.CreateInstance(type); } catch (Exception) { } - } - return null; + Assert.SkipUnless( + AceTestDatabase.SupportsColumnType(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "DATETIME2"), + AceTestDatabase.UnsupportedColumnTypeReason("DATETIME2")); + object? engine = AceTestDatabase.CreateDaoEngine(); + Assert.SkipWhen(engine is null, "DAO is unavailable in this process."); + return engine!; } } diff --git a/test/LibRed.Engine.AccessTests/FunctionArityAccessTests.cs b/test/LibRed.Engine.AccessTests/FunctionArityAccessTests.cs index 8f9cffa8e..d8818fcf5 100644 --- a/test/LibRed.Engine.AccessTests/FunctionArityAccessTests.cs +++ b/test/LibRed.Engine.AccessTests/FunctionArityAccessTests.cs @@ -17,10 +17,12 @@ private sealed record Arity(string Name, int Min, int? Max, params string[] Argu private static readonly Arity[] ConversionMathString = [ .. Unary("CBool", "CByte", "CInt", "CLng", "CSng", "CDbl", "CCur", "CStr", "CDate", "CVar", - "Abs", "Sgn", "Int", "Fix", "Sqr", "Exp", "Log", "Sin", "Cos", "Tan", "Atn", + "Abs", "Sgn", "Int", "Fix", "Sqr", "Exp", "Sin", "Cos", "Tan", "Atn", "Len", "LCase", "UCase", "Trim", "LTrim", "RTrim", "Space", "StrReverse", "Str", "Val", "Chr", "Asc", "Hex", "Oct"), new("Round", 1, 2, "1", "0", "0"), new("Rnd", 0, 1, "1", "1"), new("Timer", 0, 0, "1"), + // ACE's Log takes one argument; LibRed's also takes the standard's base as a second (docs/functions.md). + new("Log", 1, 2, "1", "1", "1"), new("Left", 2, 2, "'abc'", "1", "0"), new("Right", 2, 2, "'abc'", "1", "0"), new("Mid", 2, 3, "'abc'", "1", "1", "0"), new("InStr", 2, 4, "1", "'abc'", "'b'", "0", "0"), diff --git a/test/LibRed.Engine.AccessTests/IndexUniqueEntryCountAccessTests.cs b/test/LibRed.Engine.AccessTests/IndexUniqueEntryCountAccessTests.cs new file mode 100644 index 000000000..66b5fca50 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/IndexUniqueEntryCountAccessTests.cs @@ -0,0 +1,229 @@ +using System.Data.OleDb; +using LibRed; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// An index's unique-entry count (+4 of its statistics block) advances on INSERT only, by one when the row brings a +/// key the index does not hold yet — collation-equal keys are one key, a Null is a key except in an IGNORE NULL +/// index, a key whose last row was deleted counts again, and an UPDATE never advances it +/// (docs/format/page-02d-constraints.md §3.3.1). The same statements through ACE and LibRed leave the same counts. +/// +[Collection(AceCollection.Name)] +public class IndexUniqueEntryCountAccessTests(ITestOutputHelper output) +{ + private static readonly string[] Setup = + [ + "CREATE TABLE S (Id LONG CONSTRAINT pkS PRIMARY KEY, A LONG, T TEXT(10), N LONG)", + "CREATE INDEX ixA ON S (A)", + "CREATE INDEX ixT ON S (T)", + "CREATE INDEX ixN ON S (N)", + "CREATE INDEX ixAN ON S (A, N)", + "CREATE INDEX ixNi ON S (N) WITH IGNORE NULL", + ]; + + private static readonly string[] Steps = + [ + "INSERT INTO S VALUES (1, 5, 'a', NULL)", + "INSERT INTO S VALUES (2, 5, 'A', NULL)", + "INSERT INTO S VALUES (3, 6, 'b', 1)", + "INSERT INTO S VALUES (4, 6, 'b', 1)", + "DELETE FROM S WHERE Id = 3", + "DELETE FROM S WHERE Id = 4", + "INSERT INTO S VALUES (5, 6, 'b', 1)", + "UPDATE S SET A = 7 WHERE Id = 1", + "UPDATE S SET T = 'z' WHERE Id = 2", + ]; + + [Fact] + public void Libred_counts_new_keys_as_ace_does() + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + string ace = TemporaryDatabase.CopyPath(northwind, "uniqcount-ace-"); + string libred = TemporaryDatabase.CopyPath(northwind, "uniqcount-lib-"); + try + { + foreach (string sql in Setup) { Ace(ace, sql); LibRed(libred, sql); } + foreach (string sql in Steps) + { + Ace(ace, sql); + LibRed(libred, sql); + string aceCounts = Counts(ace), libredCounts = Counts(libred); + output.WriteLine($"{sql,-40} ACE {aceCounts} LibRed {libredCounts}"); + Assert.Equal(aceCounts, libredCounts); + } + Assert.Equal("ixA=3 ixAN=3 ixN=3 ixNi=2 ixT=3 pkS=5", Counts(libred)); + } + finally + { + TemporaryDatabase.Delete(ace); + TemporaryDatabase.Delete(libred); + } + } + + // A parent's primary key also carries the relationship's incoming logical index: one real index, so one count + // per insert. The child's foreign-key index is non-unique, so a repeated parent id adds nothing. + [Fact] + public void A_real_index_shared_by_a_relationship_is_counted_once() + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + string ace = TemporaryDatabase.CopyPath(northwind, "uniqcount-rel-ace-"); + string libred = TemporaryDatabase.CopyPath(northwind, "uniqcount-rel-lib-"); + string[] statements = + [ + "CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY)", + "CREATE TABLE C (Id LONG, PId LONG CONSTRAINT fkCP REFERENCES P (Id))", + "INSERT INTO P VALUES (1)", "INSERT INTO P VALUES (2)", "INSERT INTO P VALUES (3)", + "INSERT INTO C VALUES (10, 1)", "INSERT INTO C VALUES (11, 1)", "INSERT INTO C VALUES (12, 2)", + ]; + try + { + foreach (string sql in statements) { Ace(ace, sql); LibRed(libred, sql); } + string aceCounts = Counts(ace, "P") + " / " + Counts(ace, "C"); + string libredCounts = Counts(libred, "P") + " / " + Counts(libred, "C"); + output.WriteLine($"ACE {aceCounts} LibRed {libredCounts}"); + Assert.Equal(aceCounts, libredCounts); + } + finally + { + TemporaryDatabase.Delete(ace); + TemporaryDatabase.Delete(libred); + } + } + + // Building an index over existing rows sets its total to the entries it holds and its unique count to its + // distinct keys, from the rows present — here after deletions, so they differ from the cumulative counts the + // table's other indexes keep; no other DDL touches those. + [Theory] + [InlineData("CREATE INDEX ixN ON S (N)")] + [InlineData("CREATE INDEX ixNi ON S (N) WITH IGNORE NULL")] + [InlineData("CREATE INDEX ixAN ON S (A, N)")] + [InlineData("CREATE UNIQUE INDEX uxU ON S (U)")] + [InlineData("ALTER TABLE S ADD CONSTRAINT fkSP FOREIGN KEY (W) REFERENCES P (Id)")] + [InlineData("ALTER TABLE S ALTER COLUMN A DOUBLE")] + [InlineData("ALTER TABLE S ALTER COLUMN U DOUBLE")] + [InlineData("ALTER TABLE S ADD COLUMN Z LONG")] + [InlineData("ALTER TABLE S DROP COLUMN U")] + [InlineData("DROP INDEX ixA ON S")] + public void An_index_built_over_existing_rows_is_counted_from_them(string sql) + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + string start = TemporaryDatabase.CopyPath(northwind, "uniqcount-ddl-start-"); + string ace = "", libred = ""; + try + { + foreach (string statement in (string[]) + [ + "CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY)", + "INSERT INTO P VALUES (1)", "INSERT INTO P VALUES (2)", "INSERT INTO P VALUES (3)", + "CREATE TABLE S (Id LONG CONSTRAINT pkS PRIMARY KEY, A LONG, T TEXT(10), N LONG, U LONG, W LONG)", + "CREATE INDEX ixA ON S (A)", "CREATE INDEX ixT ON S (T)", + "INSERT INTO S VALUES (1, 5, 'a', NULL, 10, 1)", "INSERT INTO S VALUES (2, 5, 'A', NULL, 20, 1)", + "INSERT INTO S VALUES (3, 6, 'b', 1, 30, 2)", "INSERT INTO S VALUES (4, 6, 'b', 1, 40, 3)", + "INSERT INTO S VALUES (5, 7, 'c', 2, 50, 3)", "INSERT INTO S VALUES (6, 8, 'd', NULL, 60, 2)", + "DELETE FROM S WHERE Id = 5", "DELETE FROM S WHERE Id = 2", + ]) + Ace(start, statement); + + ace = TemporaryDatabase.CopyPath(start, "uniqcount-ddl-ace-"); + libred = TemporaryDatabase.CopyPath(start, "uniqcount-ddl-lib-"); + Ace(ace, sql); + LibRed(libred, sql); + string aceStats = Statistics(ace), libredStats = Statistics(libred); + output.WriteLine($"{sql}\n ACE {aceStats}\n LibRed {libredStats}"); + Assert.Equal(aceStats, libredStats); + } + finally + { + TemporaryDatabase.Delete(start); + if (ace.Length > 0) TemporaryDatabase.Delete(ace); + if (libred.Length > 0) TemporaryDatabase.Delete(libred); + } + } + + // A retype to or from Memo/OLE, which LibRed does by rebuilding the whole table, leaves the counts as ACE's ALTER + // does: only an index over the changed column is rebuilt (a primary key included), every other index here keeps + // its cumulative counts, and so does the foreign-key index of a table referencing this one. + [Theory] + [InlineData("ALTER TABLE S ALTER COLUMN U MEMO")] + [InlineData("ALTER TABLE S ALTER COLUMN T MEMO")] + [InlineData("ALTER TABLE S ALTER COLUMN M TEXT(20)")] + [InlineData("ALTER TABLE Q ALTER COLUMN Code MEMO")] + public void A_memo_or_ole_retype_recounts_only_the_index_over_the_column(string sql) + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + string start = TemporaryDatabase.CopyPath(northwind, "uniqcount-memo-start-"); + string ace = "", libred = ""; + try + { + foreach (string statement in (string[]) + [ + "CREATE TABLE S (Id LONG CONSTRAINT pkS PRIMARY KEY, A LONG, T TEXT(10), U LONG, M MEMO)", + "CREATE INDEX ixA ON S (A)", "CREATE INDEX ixT ON S (T)", "CREATE INDEX ixM ON S (M)", + "INSERT INTO S VALUES (1, 5, 'a', 10, 'm1')", "INSERT INTO S VALUES (2, 5, 'A', 20, 'm1')", + "INSERT INTO S VALUES (3, 6, 'b', 30, 'm2')", "INSERT INTO S VALUES (4, 6, 'b', 40, 'm3')", + "INSERT INTO S VALUES (5, 7, 'c', 50, 'm3')", "INSERT INTO S VALUES (6, 8, 'd', 60, 'm4')", + "CREATE TABLE K (Id LONG CONSTRAINT pkK PRIMARY KEY, SId LONG CONSTRAINT fkKS REFERENCES S (Id))", + "INSERT INTO K VALUES (1, 1)", "INSERT INTO K VALUES (2, 1)", "INSERT INTO K VALUES (3, 3)", + "INSERT INTO K VALUES (4, 4)", "INSERT INTO K VALUES (5, 6)", + "CREATE TABLE Q (Code TEXT(10) CONSTRAINT pkQ PRIMARY KEY, V LONG)", + "INSERT INTO Q VALUES ('x', 1)", "INSERT INTO Q VALUES ('y', 2)", + "DELETE FROM S WHERE Id = 5", "DELETE FROM K WHERE Id = 2", "DELETE FROM Q WHERE Code = 'x'", + ]) + Ace(start, statement); + + ace = TemporaryDatabase.CopyPath(start, "uniqcount-memo-ace-"); + libred = TemporaryDatabase.CopyPath(start, "uniqcount-memo-lib-"); + Ace(ace, sql); + LibRed(libred, sql); + string aceStats = Statistics(ace, "S", "K", "Q"), libredStats = Statistics(libred, "S", "K", "Q"); + output.WriteLine($"{sql}\n ACE {aceStats}\n LibRed {libredStats}"); + Assert.Equal(aceStats, libredStats); + } + finally + { + TemporaryDatabase.Delete(start); + if (ace.Length > 0) TemporaryDatabase.Delete(ace); + if (libred.Length > 0) TemporaryDatabase.Delete(libred); + } + } + + /// Each of the tables' real indexes as table.name=total/unique, read from its statistics block. + private static string Statistics(string path, params string[] tables) + { + if (tables.Length == 0) tables = ["S"]; + using var db = JetDatabase.Open(path); + byte[] file = File.ReadAllBytes(path); + return string.Join(" ", tables.SelectMany(name => + { + var table = db.Catalog.FindTable(name)!; + int block = table.DefinitionPage * 4096 + 0x3F; + return table.Indexes.GroupBy(i => i.RealIndexOrdinal).Select(g => g.First()) + .OrderBy(i => i.Name, StringComparer.Ordinal) + .Select(i => $"{name}.{i.Name}={BitConverter.ToInt32(file, block + i.RealIndexOrdinal * 12)}" + + $"/{BitConverter.ToInt32(file, block + i.RealIndexOrdinal * 12 + 4)}"); + })); + } + + private static void Ace(string path, string sql) + { + using OleDbConnection connection = AceTestDatabase.Open(path); + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + private static void LibRed(string path, string sql) + { + using var db = JetDatabase.Open(path, readOnly: false); + new QueryEngine(db).ExecuteNonQuery(sql); + } + + private static string Counts(string path, string table = "S") + { + using var db = JetDatabase.Open(path); + return string.Join(" ", db.Catalog.FindTable(table)!.Indexes.OrderBy(i => i.Name, StringComparer.Ordinal) + .Select(i => $"{i.Name}={i.UniqueEntryCount}")); + } +} diff --git a/test/LibRed.Engine.AccessTests/OleIndexRefusalAccessTests.cs b/test/LibRed.Engine.AccessTests/OleIndexRefusalAccessTests.cs new file mode 100644 index 000000000..866a40454 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/OleIndexRefusalAccessTests.cs @@ -0,0 +1,77 @@ +using System.Data.OleDb; +using LibRed; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// An OLE column cannot be in an index — a key, a unique constraint, a relationship's — and ACE refuses every route +/// that would put one there, up front, with "Invalid field definition '…' in definition of index or relationship.", +/// leaving nothing behind (docs/format/page-03-04-index-btree.md). LibRed refuses the same statements at the same +/// point with the same message; it used to accept them on an empty table, after which every insert failed. +/// +[Collection(AceCollection.Name)] +public class OleIndexRefusalAccessTests(ITestOutputHelper output) +{ + [Theory] + [InlineData("CREATE TABLE X (Id LONG, O LONGBINARY)", "CREATE INDEX ixO ON X (O)")] + [InlineData("CREATE TABLE X (Id LONG, O LONGBINARY CONSTRAINT pkX PRIMARY KEY)")] + [InlineData("CREATE TABLE X (Id LONG, O LONGBINARY CONSTRAINT uxO UNIQUE)")] + [InlineData("CREATE TABLE X (Id LONG, O LONGBINARY)", "ALTER TABLE X ADD CONSTRAINT pkX PRIMARY KEY (O)")] + [InlineData("CREATE TABLE X (Id LONG, O LONGBINARY)", "ALTER TABLE X ADD CONSTRAINT uxO UNIQUE (O)")] + [InlineData("CREATE TABLE X (Id LONG, A LONG)", "CREATE INDEX ixA ON X (A)", "ALTER TABLE X ALTER COLUMN A LONGBINARY")] + [InlineData("CREATE TABLE X (Id LONG, A LONG)", "CREATE INDEX ixA ON X (A)", "INSERT INTO X VALUES (1, 5)", "ALTER TABLE X ALTER COLUMN A LONGBINARY")] + // Refused as OLE before the relationship's type match would refuse it. + [InlineData("CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY)", "CREATE TABLE X (Id LONG, O LONGBINARY)", "ALTER TABLE X ADD CONSTRAINT fkXP FOREIGN KEY (O) REFERENCES P (Id)")] + [InlineData("CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY)", "CREATE TABLE X (Id LONG, O LONGBINARY CONSTRAINT fkXP REFERENCES P (Id))")] + public void Libred_refuses_an_ole_column_in_an_index_as_ace_does(params string[] steps) + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + string ace = TemporaryDatabase.CopyPath(northwind, "oleix-ace-"); + string libred = TemporaryDatabase.CopyPath(northwind, "oleix-lib-"); + try + { + string aceOutcome = Run(steps, sql => + { + using OleDbConnection connection = AceTestDatabase.Open(ace); + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + }) + "; " + Left(ace); + string libredOutcome = Run(steps, sql => + { + using var db = JetDatabase.Open(libred, readOnly: false); + new QueryEngine(db).ExecuteNonQuery(sql); + }) + "; " + Left(libred); + + output.WriteLine($"ACE {aceOutcome}\nLibRed {libredOutcome}"); + Assert.Contains("Invalid field definition", aceOutcome); + Assert.Equal(aceOutcome, libredOutcome); + } + finally + { + TemporaryDatabase.Delete(ace); + TemporaryDatabase.Delete(libred); + } + } + + /// Runs the steps until one is refused: which step, and the refusal's message. + private static string Run(string[] steps, Action execute) + { + for (int step = 0; step < steps.Length; step++) + { + try { execute(steps[step]); } + catch (Exception e) { return $"refused at step {step + 1}: {e.GetBaseException().Message}"; } + } + return "accepted"; + } + + /// What the statements left: table X's columns and indexes, or its absence. + private static string Left(string path) + { + using var db = JetDatabase.Open(path); + var table = db.Catalog.FindTable("X"); + return table is null ? "no table X" + : $"X [{string.Join(",", table.Columns.Select(c => $"{c.Name}:{c.Type}"))}] indexes [{string.Join(",", table.Indexes.Select(i => i.Name).Order(StringComparer.Ordinal))}]"; + } +} diff --git a/test/LibRed.Engine.AccessTests/ReferencesAndIdentityAccessTests.cs b/test/LibRed.Engine.AccessTests/ReferencesAndIdentityAccessTests.cs new file mode 100644 index 000000000..1fd695730 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/ReferencesAndIdentityAccessTests.cs @@ -0,0 +1,280 @@ +using System.Data.OleDb; +using LibRed.Catalog; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Two DDL forms from Access's CONSTRAINT and CREATE TABLE syntax, run through ACE and through LibRed on copies of +/// the same file, must both be accepted or both refused, and leave the same table: the same relationships and the +/// same columns — type, AutoNumber, Required — and number the same ids on insert. +/// +/// +/// REFERENCES table with no column list references the parent's primary key, pairing columns in +/// order whatever their names. It is refused when the parent has no primary key, when the column counts differ, +/// and for a table referencing itself whose key is declared later in the statement. +/// IDENTITY [(seed [, increment])] is a column attribute, allowed after the type, NULL/NOT NULL or +/// another IDENTITY and before everything else. It makes a Long column an AutoNumber and is ignored on other +/// types. COUNTER and AUTOINCREMENT are types only: neither may trail a type the same way. +/// +[Collection(AceCollection.Name)] +public class ReferencesAndIdentityAccessTests : TempDatabaseTest +{ + private static string Northwind => Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + + private const string Parent ="CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY, Code TEXT(10) CONSTRAINT uqCode UNIQUE)"; + private const string Parent2 = "CREATE TABLE P2 (A LONG, B LONG, CONSTRAINT pkP2 PRIMARY KEY (A, B))"; + private const string NoKey = "CREATE TABLE P3 (Id LONG, Code TEXT(10) CONSTRAINT uq3 UNIQUE)"; + + public static TheoryData References => new() + { + { Parent, "CREATE TABLE C (Id LONG, PId LONG REFERENCES P)", true }, + { Parent, "CREATE TABLE C (Id LONG, PId LONG CONSTRAINT fk REFERENCES P)", true }, + { Parent, "CREATE TABLE C (Id LONG, PId LONG, CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P)", true }, + { Parent, "CREATE TABLE C (Id LONG, Code LONG, CONSTRAINT fk FOREIGN KEY (Code) REFERENCES P)", true }, + { Parent, "CREATE TABLE C (Id LONG, PId LONG, CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P ON DELETE CASCADE)", true }, + { Parent2, "CREATE TABLE C (Id LONG, X LONG, Y LONG, CONSTRAINT fk FOREIGN KEY (X, Y) REFERENCES P2)", true }, + { Parent2, "CREATE TABLE C (Id LONG, X LONG, CONSTRAINT fk FOREIGN KEY (X) REFERENCES P2)", false }, + { NoKey, "CREATE TABLE C (Id LONG, PId LONG REFERENCES P3)", false }, + { Parent, "CREATE TABLE C (Id LONG, PId LONG);ALTER TABLE C ADD CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P", true }, + { Parent, "CREATE TABLE C (Id LONG);ALTER TABLE C ADD COLUMN PId LONG REFERENCES P", true }, + { Parent, "CREATE TABLE C (Id LONG);ALTER TABLE C ADD COLUMN PId LONG CONSTRAINT fk REFERENCES P (Id)", true }, + { "", "CREATE TABLE C (Id LONG CONSTRAINT pkC PRIMARY KEY, ParentId LONG REFERENCES C)", true }, + { "", "CREATE TABLE C (Id LONG CONSTRAINT pkC PRIMARY KEY, ParentId LONG, CONSTRAINT fk FOREIGN KEY (ParentId) REFERENCES C)", true }, + { "", "CREATE TABLE C (ParentId LONG REFERENCES C, Id LONG CONSTRAINT pkC PRIMARY KEY)", false }, + { "", "CREATE TABLE C (Id LONG, ParentId LONG, CONSTRAINT fk FOREIGN KEY (ParentId) REFERENCES C, CONSTRAINT pkC PRIMARY KEY (Id))", false }, + { "", "CREATE TABLE C (Id LONG, PId LONG REFERENCES Nope)", false }, + }; + + public static TheoryData Identities => new() + { + { "Id INT NOT NULL IDENTITY", true }, + { "Id INT IDENTITY NOT NULL", true }, + { "Id INT IDENTITY", true }, + { "Id INT NOT NULL IDENTITY(5, 2)", true }, + { "Id INT NOT NULL IDENTITY (5, 2)", true }, + { "Id INT IDENTITY(5)", true }, + { "Id INT IDENTITY(5, -1)", true }, + { "Id IDENTITY NOT NULL", true }, + { "Id IDENTITY(5, 2)", true }, + { "Id LONG NOT NULL IDENTITY", true }, + { "Id INTEGER4 NOT NULL IDENTITY", true }, + { "Id INT NOT NULL IDENTITY PRIMARY KEY", true }, + { "Id INT IDENTITY DEFAULT 1", true }, + { "Id INT IDENTITY IDENTITY", true }, + { "Id COUNTER NOT NULL", true }, + { "Id COUNTER", true }, + { "Id AUTOINCREMENT(5, 2) NOT NULL", true }, + { "Id COUNTER NOT NULL IDENTITY(9, 3)", true }, + { "Id COUNTER(5, 2) IDENTITY(9, 3)", true }, + { "Id COUNTER(5, 2) IDENTITY", true }, + { "Id AUTOINCREMENT(5, 2) NOT NULL IDENTITY", true }, + // Accepted and ignored on anything but a Long. + { "Id SHORT NOT NULL IDENTITY", true }, + { "Id BYTE IDENTITY", true }, + { "Id BIGINT NOT NULL IDENTITY", true }, + { "Id DOUBLE IDENTITY", true }, + { "Id GUID IDENTITY", true }, + { "Id TEXT(10) NOT NULL IDENTITY", true }, + { "Id TEXT IDENTITY(5, 2)", true }, + // Refused by both. + { "Id INT PRIMARY KEY IDENTITY(5, 2)", false }, + { "Id INT CONSTRAINT pk PRIMARY KEY NOT NULL IDENTITY", false }, + { "Id INT DEFAULT 1 IDENTITY", false }, + { "Id INT NOT NULL IDENTITY(5, 2, 1)", false }, + { "Id INT NOT NULL IDENTITY()", false }, + { "Id INT NOT NULL AUTOINCREMENT", false }, + { "Id INT NOT NULL COUNTER", false }, + { "Identity LONG", false }, + }; + + [Theory] + [MemberData(nameof(References))] + public void References_without_a_column_list_matches_ace(string parent, string statements, bool accepted) + => AssertSameOutcome(parent.Length == 0 ? statements : $"{parent};{statements}", "C", insertColumn: null, accepted); + + [Theory] + [MemberData(nameof(Identities))] + public void Identity_matches_ace(string column, bool accepted) + { + // An ACE older than the Large Number type refuses the column as a syntax error, which says nothing about + // IDENTITY. CI's 2016 redistributable is one. + if (column.Contains("BIGINT", StringComparison.Ordinal)) + { + Assert.SkipUnless(AceTestDatabase.SupportsColumnType(Northwind, "BIGINT"), + AceTestDatabase.UnsupportedColumnTypeReason("BIGINT")); + } + AssertSameOutcome($"CREATE TABLE T ({column}, V TEXT(10))", "T", insertColumn: "V", accepted); + } + + // A relationship needs the same storage type on both sides, whatever the lengths — an AutoNumber being a Long on + // either side. Measured over every pairing of the column types; these are representative. + public static TheoryData RelationshipTypes => new() + { + { "COUNTER", "LONG", true }, + { "LONG", "LONG", true }, + { "LONG", "COUNTER", true }, + { "COUNTER", "COUNTER", true }, + { "TEXT(10)", "TEXT(20)", true }, + { "TEXT(20)", "CHAR(10)", true }, + { "DECIMAL(10, 2)", "DECIMAL(12, 4)", true }, + { "BINARY(8)", "VARBINARY(8)", true }, + { "GUID", "GUID", true }, + { "LONG", "TEXT(10)", false }, + { "SHORT", "COUNTER", false }, + { "LONG", "SHORT", false }, + { "LONG", "BIGINT", false }, + { "SHORT", "LONG", false }, + { "DOUBLE", "SINGLE", false }, + { "CURRENCY", "DECIMAL(10, 2)", false }, + { "TEXT(10)", "MEMO", false }, + { "DATETIME", "DOUBLE", false }, + }; + + [Theory] + [MemberData(nameof(RelationshipTypes))] + public void A_relationship_between_column_types_matches_ace(string parentType, string childType, bool accepted) + { + AssertSameOutcome($"CREATE TABLE P (Id {parentType} CONSTRAINT pkP PRIMARY KEY);CREATE TABLE C (K LONG, PId {childType} REFERENCES P (Id))", + "C", insertColumn: null, accepted); + AssertSameOutcome($"CREATE TABLE P (Id {parentType} CONSTRAINT pkP PRIMARY KEY);CREATE TABLE C (K LONG, PId {childType});ALTER TABLE C ADD CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P (Id)", + "C", insertColumn: null, accepted); + } + + // The constraints on an ADD COLUMN apply to the new column, as they do in CREATE TABLE. A primary key needs a + // value in every row, and a table has only one. + public static TheoryData AddColumnConstraints => new() + { + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Code TEXT(10) UNIQUE", true }, + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Code TEXT(10) CONSTRAINT uq UNIQUE", true }, + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Code TEXT(10) NOT NULL CONSTRAINT uq UNIQUE", true }, + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Id LONG PRIMARY KEY", true }, + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY", true }, + { "CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, V TEXT(10));ALTER TABLE T ADD COLUMN Id2 LONG CONSTRAINT pk2 PRIMARY KEY", false }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY", false }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Code TEXT(10) CONSTRAINT uq UNIQUE", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id COUNTER CONSTRAINT pk PRIMARY KEY", true }, + // Existing rows are numbered 1, 2, …; the default counter carries on after them, any other restarts at its seed. + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id COUNTER(5, 2)", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id COUNTER(1, 1)", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id INT IDENTITY", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id COUNTER(2, 1)", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');ALTER TABLE T ADD COLUMN Id COUNTER(1, 5)", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');INSERT INTO T (V) VALUES ('c');ALTER TABLE T ADD COLUMN Id COUNTER(10, -1)", true }, + { "CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');INSERT INTO T (V) VALUES ('b');INSERT INTO T (V) VALUES ('c');DELETE FROM T WHERE V = 'a';ALTER TABLE T ADD COLUMN Id COUNTER", true }, + { "CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Id COUNTER(5, 2)", true }, + { "CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY);CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN PId LONG CONSTRAINT uq UNIQUE REFERENCES P", true }, + // The same primary-key rules through the other two ways of adding one. + { "CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, V TEXT(10), W LONG);ALTER TABLE T ADD CONSTRAINT pk2 PRIMARY KEY (W)", false }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');ALTER TABLE T ADD CONSTRAINT pk PRIMARY KEY (W)", false }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX pk ON T (W) WITH PRIMARY", false }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V, W) VALUES ('a', 1);CREATE INDEX pk ON T (W) WITH PRIMARY", true }, + // WITH DISALLOW NULL needs a value in every key column too; IGNORE NULL does not. + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX ix ON T (W) WITH DISALLOW NULL", false }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX ix ON T (V, W) WITH DISALLOW NULL", false }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V, W) VALUES ('a', 1);CREATE INDEX ix ON T (W) WITH DISALLOW NULL", true }, + { "CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX ix ON T (W) WITH IGNORE NULL", true }, + }; + + [Theory] + [MemberData(nameof(AddColumnConstraints))] + public void Add_column_constraints_match_ace(string statements, bool accepted) + => AssertSameOutcome(statements, "T", insertColumn: "V", accepted); + + [Fact] + public void Identity_on_alter_table_matches_ace() + { + AssertSameOutcome("CREATE TABLE T (V TEXT(10));ALTER TABLE T ADD COLUMN Id INT NOT NULL IDENTITY", "T", "V", accepted: true); + AssertSameOutcome("CREATE TABLE T (Id INT, V TEXT(10));ALTER TABLE T ALTER COLUMN Id INT IDENTITY", "T", "V", accepted: true); + } + + private static void AssertSameOutcome(string statements, string table, string? insertColumn, bool? accepted) + { + string[] sql = statements.Split(';'); + (string ace, string? aceError) = Run(sql, table, insertColumn, (path, statement) => + { + using OleDbConnection connection = AceTestDatabase.Open(path); + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = statement; + command.ExecuteNonQuery(); + }); + (string libred, string? libredError) = Run(sql, table, insertColumn, (path, statement) => + { + using var db = JetDatabase.Open(path, readOnly: false); + new QueryEngine(db).ExecuteNonQuery(statement); + }); + + // Say which side refused and why: a bare True/False cannot tell a real difference from a statement this ACE + // cannot run at all (a type it predates, say). + if (accepted is bool expected && expected != (aceError is null)) + { + Assert.Fail(expected + ? $"ACE refused what the test expects it to accept. {aceError}" + : $"ACE accepted what the test expects it to refuse: {ace}"); + } + if (ace != libred) + { + Assert.Fail($"LibRed and ACE differ.{Environment.NewLine}" + + $"ACE: {ace} {aceError}{Environment.NewLine}" + + $"LibRed: {libred} {libredError}"); + } + } + + /// Runs the statements on a fresh copy and describes the result: refused, with the statement and error + /// that refused it, or the table's columns and relationships as LibRed's catalog reads them, then the ids three + /// inserts receive. + private static (string Description, string? Error) Run(string[] sql, string table, string? insertColumn, + Action execute) + { + string path = TemporaryDatabase.CopyPath(Northwind, "refident-"); + foreach (string statement in sql) + { + try + { + execute(path, statement); + } + catch (Exception e) + { + return ("refused", $"[{statement}] {e.GetType().Name}: {e.Message}"); + } + } + + var description = new List(); + string? counter; + using (var db = JetDatabase.Open(path)) + { + TableDef t = db.Catalog.FindTable(table)!; + description.AddRange(t.Columns.Select(c => + $"{c.Name} {c.Type} autonumber={c.IsAutoNumber} required={!c.IsNullable}")); + description.AddRange(db.Catalog.ForeignKeysOf(table).Select(fk => + $"relation {fk.Table}->{fk.ReferencedTable} [{string.Join(", ", fk.Columns.Select(c => $"{c.Column}={c.ReferencedColumn}"))}] cascadeDelete={fk.CascadeDelete}")); + // Names are left out: ACE gives an unnamed constraint's index a random one. + description.AddRange(t.Indexes + .Select(ix => $"index primary={ix.IsPrimaryKey} unique={ix.IsUnique} [{string.Join(",", ix.Columns.Select(c => c.Column.Name))}]") + .Order(StringComparer.Ordinal)); + counter = t.Columns.FirstOrDefault(c => c.IsAutoNumber)?.Name; + } + + if (counter is not null && insertColumn is not null) + { + using (OleDbConnection connection = AceTestDatabase.Open(path)) + { + for (int i = 0; i < 3; i++) + { + using OleDbCommand insert = connection.CreateCommand(); + insert.CommandText = $"INSERT INTO [{table}] ([{insertColumn}]) VALUES (NULL)"; + insert.ExecuteNonQuery(); + } + using OleDbCommand read = connection.CreateCommand(); + read.CommandText = $"SELECT [{counter}] FROM [{table}] ORDER BY [{counter}]"; + using OleDbDataReader reader = read.ExecuteReader(); + var ids = new List(); + while (reader.Read()) ids.Add(Convert.ToString(reader.GetValue(0))!); + description.Add($"ids {string.Join(",", ids)}"); + } + } + + return (string.Join("; ", description), null); + } +} diff --git a/test/LibRed.Engine.AccessTests/StatementAtomicityTests.cs b/test/LibRed.Engine.AccessTests/StatementAtomicityTests.cs index 4a91b77bc..6688eca94 100644 --- a/test/LibRed.Engine.AccessTests/StatementAtomicityTests.cs +++ b/test/LibRed.Engine.AccessTests/StatementAtomicityTests.cs @@ -80,7 +80,7 @@ public void Failed_multirow_update_rolls_back_grown_index_keys_byte_for_byte_and var e = new QueryEngine(db); e.ExecuteNonQuery( "CREATE TABLE AtomicSplit (Id LONG PRIMARY KEY, Code TEXT(100), " + - "CONSTRAINT CK_Last CHECK (Id < 900 OR Code NOT LIKE 'expanded-*'))"); + "CONSTRAINT CK_Last CHECK (Id < 900 OR Code NOT LIKE 'expanded-%'))"); e.ExecuteNonQuery("CREATE UNIQUE INDEX UX_AtomicSplit_Code ON AtomicSplit (Code)"); for (int i = 1; i <= 900; i++) e.ExecuteNonQuery($"INSERT INTO AtomicSplit (Id, Code) VALUES ({i}, 'k{i}')"); @@ -100,7 +100,7 @@ public void Failed_multirow_update_rolls_back_grown_index_keys_byte_for_byte_and Assert.Equal(before, File.ReadAllBytes(path)); using var connection = AceTestDatabase.Open(path); AssertScalar(connection, "SELECT COUNT(*) FROM AtomicSplit", 900); - AssertScalar(connection, "SELECT COUNT(*) FROM AtomicSplit WHERE Code LIKE 'expanded-*'", 0); + AssertScalar(connection, "SELECT COUNT(*) FROM AtomicSplit WHERE Code LIKE 'expanded-%'", 0); AssertScalar(connection, "SELECT COUNT(*) FROM AtomicSplit WHERE Id = 900 AND Code = 'k900'", 1); } finally { TemporaryDatabase.Delete(path); } diff --git a/test/LibRed.Engine.AccessTests/UnionColumnTypeAccessTests.cs b/test/LibRed.Engine.AccessTests/UnionColumnTypeAccessTests.cs new file mode 100644 index 000000000..70330fe19 --- /dev/null +++ b/test/LibRed.Engine.AccessTests/UnionColumnTypeAccessTests.cs @@ -0,0 +1,140 @@ +using System.Data.OleDb; +using System.Globalization; +using LibRed; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// A UNION's column type, and every value converted to it, run through ACE and through LibRed on copies of the same +/// table: a column of each kind, paired with every other. The types must match and so must the values. +/// +/// +/// A date written as text follows ACE's regional settings (Windows') and LibRed's culture (the test's), so for a pair +/// that turns a date into text only the type is compared. The Large Number rows need an ACE that has the type, which +/// CI's 2016 redistributable does not; they are skipped there. +/// +[Collection(AceCollection.Name)] +public class UnionColumnTypeAccessTests(UnionColumnTypeAccessTests.Databases databases) + : TempDatabaseTest, IClassFixture +{ + public sealed class Databases : IDisposable + { + private const string Create = + "CREATE TABLE T (Id INT, M CURRENCY, B BYTE, S SMALLINT, D DATETIME, F DOUBLE, R REAL, E DECIMAL(18,4), " + + "L LONG, X TEXT(10), Y YESNO, G GUID, N BINARY(4))"; + private const string Insert = + "INSERT INTO T (Id, M, B, S, D, F, R, E, L, X, Y) " + + "VALUES (1, 10.5, 3, 7, #2020-01-02#, 2.5, 1.5, 4.25, 70000, 'abc', TRUE)"; + private const string Guid = "00112233-4455-6677-8899-AABBCCDDEEFF"; + + private readonly TemporaryDatabase _ace; + private readonly TemporaryDatabase _libred; + + public Databases() + { + string northwind = Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + BigInt = AceTestDatabase.SupportsColumnType(northwind, "BIGINT"); + string[] bigInt = BigInt ? ["ALTER TABLE T ADD COLUMN Z BIGINT", "UPDATE T SET Z = 5000000000"] : []; + + _ace = TemporaryDatabase.CopyOf(northwind, "union-types-ace-"); + using (OleDbConnection connection = AceTestDatabase.Open(_ace.Path)) + { + foreach (string statement in (string[])[Create, Insert, $"UPDATE T SET G = {{guid {{{Guid}}}}}", + "UPDATE T SET N = 0x41004200", .. bigInt]) + { + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = statement; + command.ExecuteNonQuery(); + } + } + + _libred = TemporaryDatabase.CopyOf(northwind, "union-types-libred-"); + using (var db = JetDatabase.Open(_libred.Path, readOnly: false)) + { + var engine = new QueryEngine(db); + foreach (string statement in (string[])[Create, Insert, $"UPDATE T SET G = {{{Guid}}}", + "UPDATE T SET N = 0x41004200", .. bigInt]) + engine.ExecuteNonQuery(statement); + } + } + + /// Whether the installed ACE has the Large Number type, and so the table has column Z. + public bool BigInt { get; } + + public (string Type, string[] Values) Ace(string sql) + { + using OleDbConnection connection = AceTestDatabase.Open(_ace.Path); + using OleDbCommand command = connection.CreateCommand(); + command.CommandText = sql; + using OleDbDataReader reader = command.ExecuteReader(); + var values = new List(); + while (reader.Read()) + values.Add(Describe(reader.GetValue(0))); + return (reader.GetFieldType(0).Name, values.ToArray()); + } + + public (string Type, string[] Values) LibRed(string sql) + { + using var db = JetDatabase.Open(_libred.Path, readOnly: true); + var result = new QueryEngine(db).ExecuteQuery(sql); + return (result.ColumnTypes[0].Name, result.Rows.Select(row => Describe(row[0])).ToArray()); + } + + /// A value with its type; a Decimal without trailing zeros, which ACE adds for a column's scale. + private static string Describe(object? value) => value switch + { + null or DBNull => "NULL", + byte[] bytes => "Byte[] " + Convert.ToHexString(bytes), + decimal m => "Decimal " + m.ToString("G29", CultureInfo.InvariantCulture), + double d => "Double " + d.ToString("R", CultureInfo.InvariantCulture), + float f => "Single " + f.ToString("R", CultureInfo.InvariantCulture), + _ => $"{value.GetType().Name} {Convert.ToString(value, CultureInfo.InvariantCulture)}", + }; + + public void Dispose() + { + _ace.Dispose(); + _libred.Dispose(); + } + } + + // Left, right, and whether the values are compared: not when a date becomes text, nor where LibRed departs. + public static TheoryData Pairs => new() + { + { "B", "5", true }, { "5", "B", true }, { "S", "70000", true }, { "B", "S", true }, + { "NULL", "B", true }, { "B", "NULL", true }, + { "M", "F", true }, { "F", "M", true }, { "E", "F", true }, { "M", "E", true }, { "E", "M", true }, + { "M", "L", true }, { "E", "L", true }, + { "R", "S", true }, { "R", "L", true }, { "L", "F", true }, + { "Y", "B", true }, { "Y", "S", true }, { "Y", "F", true }, { "Y", "M", true }, { "Y", "Y", true }, + { "X", "B", true }, { "B", "X", true }, { "Y", "X", true }, + { "D", "D", true }, { "D", "X", false }, { "D", "F", false }, { "D", "M", false }, { "Y", "D", false }, + { "G", "G", true }, { "G", "X", true }, { "X", "G", true }, { "G", "B", true }, { "G", "N", true }, + { "N", "X", true }, { "N", "B", true }, { "N", "S", true }, { "N", "L", true }, { "N", "F", true }, + { "N", "R", true }, { "N", "D", true }, { "N", "Y", true }, { "N", "M", true }, { "N", "E", true }, + // The Large Number. + { "Z", "B", true }, { "Z", "S", true }, { "L", "Z", true }, { "Z", "Y", true }, { "Z", "5", true }, + { "Z", "R", true }, { "Z", "F", true }, { "Z", "M", true }, { "M", "Z", true }, { "Z", "E", true }, + { "E", "Z", true }, { "Z", "X", true }, { "Z", "D", true }, { "NULL", "Z", true }, { "Z", "Z", true }, + // ACE cuts a Large Number's text to 8 bytes in a binary column; LibRed keeps all of it. + { "N", "Z", false }, + }; + + [Theory] + [MemberData(nameof(Pairs))] + public void A_union_column_matches_ace(string left, string right, bool compareValues) + { + if (left == "Z" || right == "Z") + Assert.SkipUnless(databases.BigInt, AceTestDatabase.UnsupportedColumnTypeReason("BIGINT")); + + string sql = $"SELECT {left} AS c FROM T WHERE Id = 1 UNION ALL SELECT {right} FROM T WHERE Id = 1"; + (string aceType, string[] aceValues) = databases.Ace(sql); + (string libredType, string[] libredValues) = databases.LibRed(sql); + + Assert.Equal(aceType, libredType); + if (compareValues) + Assert.Equal(aceValues, libredValues); + } +} diff --git a/test/LibRed.Engine.Tests/AggregateFunctionTests.cs b/test/LibRed.Engine.Tests/AggregateFunctionTests.cs new file mode 100644 index 000000000..99390c9ed --- /dev/null +++ b/test/LibRed.Engine.Tests/AggregateFunctionTests.cs @@ -0,0 +1,130 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The aggregates Sum Avg Min Max StDev StDevP Var VarP over +/// columns of every kind. The expected values were measured against ACE; result types follow LibRed's contract (as +/// LINQ's aggregates type them) rather than ACE's widened types. +/// +public class AggregateFunctionTests(AggregateFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE A (Id LONG, I LONG, S SHORT, B BYTE, R REAL, F FLOAT, C CURRENCY, D DECIMAL(18,4), T TEXT(20), " + + "TN TEXT(20), DT DATETIME, Y YESNO, G GUID, N LONG, TM TEXT(20))", + "INSERT INTO A (Id, I, S, B, R, F, C, D, T, TN, DT, Y, TM) " + + "VALUES (1, 1, 1, 1, 1.5, 2.5, 1.25, 1.5, 'b', '10', #2020-01-02 12:00#, TRUE, '5')", + "INSERT INTO A (Id, I, S, B, R, F, C, D, T, TN, DT, Y, TM) " + + "VALUES (2, 2, -3, 255, 0.1, -1.5, 2.5, 2.25, 'A', '2', #1999-12-31 13:00#, FALSE, 'x')", + "INSERT INTO A (Id) VALUES (3)", + "INSERT INTO A (Id, I, S, B, R, F, C, D, T, TN, DT, Y, TM) " + + "VALUES (4, 2, 5, 0, -2, 1E-10, -0.0001, -1, 'a', '1e3', #1899-12-29 06:00#, TRUE, '')", + "INSERT INTO A (Id, TM) VALUES (5, '7')", + "UPDATE A SET G = {00112233-4455-6677-8899-AABBCCDDEEFF} WHERE Id = 1", + ]; + + public sealed class Database() : SharedDatabase("aggregate-", Setup); + + private object? Scalar(string select) => database.Scalar($"SELECT {select}", CultureInfo.GetCultureInfo("en-US")); + + [Theory] + [InlineData("SUM(I) FROM A", 5)] + [InlineData("SUM(S) FROM A", 3)] + [InlineData("SUM(B) FROM A", 256)] + [InlineData("SUM(Y) FROM A", -2)] + [InlineData("SUM(1) FROM A", 5)] + [InlineData("COUNT(*) FROM A WHERE Id > 99", 0)] + [InlineData("COUNT(TM) FROM A", 4)] + public void Whole_number_sums_are_integers(string select, int expected) => + Assert.Equal(expected, Assert.IsType(Scalar(select))); + + [Theory] + [InlineData("SUM(TN) FROM A", 1012.0)] + [InlineData("SUM('3') FROM A", 15.0)] + [InlineData("SUM(DT) FROM A", 80356.79166666666)] + [InlineData("AVG(DT) FROM A", 26785.59722222222)] + [InlineData("AVG(Y) FROM A", -0.4)] + [InlineData("AVG(TRUE) FROM A", -1.0)] + [InlineData("SUM(F) FROM A WHERE Id IN (1, 2)", 1.0)] + public void Text_dates_and_booleans_sum_as_their_numbers(string select, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(select))); + + [Fact] + public void A_currency_sum_is_exact() => + Assert.Equal(3.7499m, Scalar("SUM(C) FROM A")); + + [Theory] + [InlineData("STDEV(I) FROM A", 0.5773502691896257)] + [InlineData("STDEVP(I) FROM A", 0.4714045207910317)] + [InlineData("VAR(S) FROM A", 16.0)] + [InlineData("VARP(S) FROM A", 10.666666666666666)] + [InlineData("VAR(B) FROM A", 21590.333333333332)] + [InlineData("VARP(B) FROM A", 14393.555555555555)] + [InlineData("VAR(R) FROM A", 3.1033333338859177)] + [InlineData("VARP(R) FROM A", 2.0688888892572788)] + [InlineData("STDEV(R) FROM A", 1.7616280350533473)] + [InlineData("STDEVP(R) FROM A", 1.4383632674874864)] + [InlineData("VAR(F) FROM A", 4.0833333333)] + [InlineData("VARP(F) FROM A", 2.7222222222)] + [InlineData("STDEVP(F) FROM A", 1.6499158227618766)] + [InlineData("VAR(C) FROM A", 1.5626166666666668)] + [InlineData("VARP(C) FROM A", 1.0417444444444444)] + [InlineData("STDEV(C) FROM A", 1.2500466657955882)] + [InlineData("STDEVP(C) FROM A", 1.0206588286222014)] + [InlineData("VAR(D) FROM A", 2.8958333333333335)] + [InlineData("VARP(D) FROM A", 1.9305555555555556)] + [InlineData("STDEV(D) FROM A", 1.7017148213885114)] + [InlineData("VAR(TN) FROM A", 329361.3333333333)] + [InlineData("VARP(TN) FROM A", 219574.22222222222)] + [InlineData("STDEV(TN) FROM A", 573.9001074519269)] + [InlineData("STDEVP(TN) FROM A", 468.58747552855294)] + [InlineData("VAR(DT) FROM A", 551499298.1012732)] + [InlineData("VARP(DT) FROM A", 367666198.7341821)] + [InlineData("STDEV(DT) FROM A", 23484.022187463397)] + [InlineData("VAR(Y) FROM A", 0.3)] + [InlineData("VARP(Y) FROM A", 0.24)] + [InlineData("STDEV(Y) FROM A", 0.5477225575051661)] + [InlineData("VAR(F) FROM A WHERE Id IN (1, 2)", 8.0)] + [InlineData("STDEVP(I) FROM A WHERE Id = 1", 0.0)] + [InlineData("VARP(I) FROM A WHERE Id = 1", 0.0)] + public void Statistics_match_ace_to_the_last_bit(string select, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(select))); + + [Theory] + [InlineData("MIN(TM) FROM A", "7")] + [InlineData("MAX(TM) FROM A", "x")] + [InlineData("MAX(TM) FROM A WHERE Id IN (4, 5)", "7")] + [InlineData("MIN(TM) FROM A WHERE Id IN (2, 4, 5)", "7")] + [InlineData("MIN(IIF(Id = 4, 'a', TM)) FROM A", "5")] + [InlineData("MIN(T) FROM A", "A")] + [InlineData("MAX(T) FROM A", "b")] + public void Empty_text_counts_as_no_value_once_it_wins(string select, string expected) => + Assert.Equal(expected, Scalar(select)); + + [Theory] + [InlineData("MIN(TM) FROM A WHERE Id IN (1, 4)")] + [InlineData("MIN(TM) FROM A WHERE Id = 4")] + [InlineData("MIN(TM) FROM A WHERE Id IN (1, 2, 4)")] + [InlineData("SUM(I) FROM A WHERE Id > 99")] + [InlineData("AVG(I) FROM A WHERE Id > 99")] + [InlineData("MAX(DT) FROM A WHERE Id > 99")] + [InlineData("SUM(N) FROM A")] + [InlineData("VAR(N) FROM A")] + [InlineData("STDEV(I) FROM A WHERE Id = 1")] + [InlineData("VAR(I) FROM A WHERE Id = 1")] + public void Null_results(string select) => + Assert.Null(Scalar(select)); + + [Theory] + [InlineData("SUM(T) FROM A")] + [InlineData("AVG(TM) FROM A")] + [InlineData("STDEV(T) FROM A")] + [InlineData("SUM(G) FROM A")] + [InlineData("VAR(G) FROM A")] + public void Values_that_are_not_numbers_are_a_type_mismatch(string select) => + Assert.Throws(() => Scalar(select)); +} diff --git a/test/LibRed.Engine.Tests/ArithmeticOperatorTests.cs b/test/LibRed.Engine.Tests/ArithmeticOperatorTests.cs new file mode 100644 index 000000000..0b34a533f --- /dev/null +++ b/test/LibRed.Engine.Tests/ArithmeticOperatorTests.cs @@ -0,0 +1,229 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The - * / \ MOD and ^ operators as ACE evaluates them: their precedence, +/// how they read text and dates, their errors, and the decimal places ACE keeps in a Decimal result. The expected +/// values were measured against ACE. +/// +public class ArithmeticOperatorTests(ArithmeticOperatorTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, N LONG, SI SHORT, SG REAL, CY CURRENCY, DC DECIMAL(18,4), D DATETIME, " + + "S TEXT(60), NT TEXT(60), G GUID, B BINARY(4))", + "INSERT INTO T (Id, N, SI, SG, CY, DC, D, S) VALUES (1, 3, 2, 1.5, 3.25, 4.5, #2020-01-02 12:00:00#, 'abc')", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("arith-ops-", Setup); + + private object? Scalar(string expression) => Query($"SELECT {expression} FROM T"); + + // Text is read in the regional separators and currency symbol, so each query runs under en-US whatever the + // machine's culture. + private object? Query(string sql) => database.Scalar(sql, EnUs); + + [Theory] + [InlineData("7 \\ 2 * 3", 1)] + [InlineData("7 \\ 2 / 2", 7)] + [InlineData("5 MOD 3 * 2", 5)] + [InlineData("10 MOD 4 \\ 2", 0)] + [InlineData("10 \\ 4 MOD 3", 2)] + [InlineData("2 * 3 MOD 4", 2)] + [InlineData("2 + 3 MOD 2", 3)] + [InlineData("8 \\ 4 \\ 2", 1)] + [InlineData("1 - 2 - 3", -4)] + public void Integer_division_and_mod_each_bind_looser_than_the_one_before(string expression, int expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("-2 ^ 2", 4d)] + [InlineData("2 ^ 3 ^ 2", 64d)] + [InlineData("3 * 2 ^ 2", 12d)] + [InlineData("8 / 4 / 2", 1d)] + public void Power_binds_tighter_than_multiplication_but_looser_than_negation(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("'2.5' - N", -0.5d)] + [InlineData("'1e2' - 1", 99d)] + [InlineData("'5-' - 1", -6d)] + [InlineData("'&H10' * 2", 32d)] + [InlineData("'$5' / 2", 2.5d)] + [InlineData("'2' ^ '3'", 8d)] + [InlineData("TRUE - '1'", -2d)] + public void Text_is_read_as_a_number(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("'(5)' \\ 2", -2)] + [InlineData("'1,000' MOD 7", 6)] + [InlineData("'2.5' \\ 1", 2)] + [InlineData("'3.5' MOD 2", 0)] + public void Text_is_read_as_a_number_before_rounding_for_integer_division(string expression, int expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("'abc' - 1")] + [InlineData("' ' * 1")] + [InlineData("'abc' * NULL")] + [InlineData("NULL - S")] + [InlineData("G / NULL")] + [InlineData("B MOD NULL")] + [InlineData("NULL ^ G")] + [InlineData("N \\ G")] + [InlineData("D - B")] + public void Text_that_is_not_a_number_or_a_guid_or_binary_value_is_a_type_mismatch_even_with_null(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("'1' * NULL")] + [InlineData("NT - 1")] + [InlineData("NULL \\ '1'")] + public void Null_propagates_once_the_operands_are_numbers(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("D / 2", 21916.25d)] + [InlineData("D ^ 1", 43832.5d)] + [InlineData("D * 2", 87665d)] + [InlineData("D - D", 0d)] + [InlineData("D - #2020-01-01#", 1.5d)] + public void A_date_is_its_serial_where_the_result_is_a_number(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("D \\ 2", 21916)] + [InlineData("D MOD 7", 5)] + [InlineData("1 MOD D", 1)] + public void A_date_rounds_to_its_serial_for_integer_division(string expression, int expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("D - 1", "2020-01-01T12:00:00")] + [InlineData("1 - D", "1779-12-28T12:00:00")] + [InlineData("D - '1'", "2020-01-01T12:00:00")] + [InlineData("'1' - D", "1779-12-28T12:00:00")] + public void Subtracting_a_number_and_a_date_is_a_date(string expression, string expected) => + Assert.Equal(DateTime.Parse(expected, CultureInfo.InvariantCulture), Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("1 / 0")] + [InlineData("0 / 0")] + [InlineData("1.5 / 0")] + [InlineData("SG / 0")] + [InlineData("DC / 0")] + [InlineData("CY / 0")] + [InlineData("1 \\ 0")] + [InlineData("1 MOD 0")] + [InlineData("0 ^ -1")] + public void Dividing_by_zero_is_an_error(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("2147483647 * 2")] + [InlineData("N * 2147483647")] + [InlineData("-2147483647 - 2")] + [InlineData("N - -2147483647")] + [InlineData("1E300 * 1E300")] + [InlineData("10 ^ 400")] + [InlineData("2 ^ 1024")] + [InlineData("'1e400' * 1")] + [InlineData("CLNG(-2147483648) \\ -1")] + [InlineData("2147483647.5 \\ 1")] + [InlineData("1E10 \\ 1")] + [InlineData("#0100-01-01# - 1")] + [InlineData("#9999-12-31# - -1")] + public void A_result_past_its_type_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("(-8) ^ (1/3)")] + [InlineData("(-2) ^ 0.5")] + public void A_negative_base_with_a_fractional_exponent_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Fact] + public void The_smallest_long_mod_minus_one_is_zero() => + Assert.Equal(0, Scalar("CLNG(-2147483648) MOD -1")); + + [Theory] + [InlineData("2.5 \\ 1", 2)] + [InlineData("3.5 \\ 1", 4)] + [InlineData("-2.5 \\ 1", -2)] + [InlineData("0.5 \\ 1", 0)] + [InlineData("-7 \\ 2", -3)] + [InlineData("7 \\ -2", -3)] + [InlineData("7.5 MOD 2", 0)] + [InlineData("19 MOD 6.7", 5)] + [InlineData("12.6 MOD 5", 3)] + [InlineData("12 MOD 4.3", 0)] + [InlineData("-7 MOD 3", -1)] + [InlineData("7 MOD -3", 1)] + [InlineData("SG \\ 1", 2)] + [InlineData("CY \\ 1", 3)] + [InlineData("DC \\ 1", 4)] + public void Integer_division_rounds_its_operands_half_to_even_and_truncates(string expression, int expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("TRUE / SG", -0.6666667f)] + [InlineData("SI / SG", 1.3333334f)] + [InlineData("CSNG(1) / CSNG(10)", 0.1f)] + public void A_single_divided_with_integers_or_booleans_stays_a_single(string expression, float expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("1 / 1.5", 0.6d)] + [InlineData("1.5 / '2.5'", 0.6d)] + [InlineData("1.5 * 1.5", 2.2d)] + [InlineData("1.5 * 1.5 * 1.5", 3.3d)] + [InlineData("459.35 / 3", 153.11d)] + [InlineData("40000 / 0.3", 133333.3d)] + [InlineData("D * 0.25", 10958.12d)] + [InlineData("1.5 / LEFT('12', 2)", 0.1d)] + [InlineData("CSTR(5) / 1.5", 3.3d)] + [InlineData("1.5 * 1.25", 1.875d)] + [InlineData("459.35 * 334.90", 153836.315d)] + [InlineData("1.1 * 1.11", 1.2210000000000003d)] + [InlineData("1 / 3.0", 0.3333333333333333d)] + public void A_number_written_with_a_decimal_point_keeps_its_places_unless_another_decimal_has_different_ones( + string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("1 / DC", "0.2222")] + [InlineData("TRUE / DC", "-0.2222")] + [InlineData("DC / 40000", "0.0001")] + [InlineData("DC / '7'", "0.6428")] + [InlineData("DC * DC", "20.2500")] + [InlineData("D / DC", "9740.5555")] + [InlineData("CY * CY", "10.5625")] + [InlineData("CY / 3", "1.0833333333333333333333333333")] + [InlineData("1.5 / DC", "0.3333333333333333333333333333")] + public void A_decimal_column_keeps_its_places_but_currency_divided_is_whole(string expression, string expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)).ToString(CultureInfo.InvariantCulture)); + + [Fact] + public void A_decimal_literal_meets_a_decimal_as_written() => + Assert.Equal(1507.05m, Assert.IsType(Scalar("DC * 334.90"))); + + [Theory] + [InlineData("SELECT d.X / 7 FROM (SELECT DC AS X FROM T) AS d", "0.6428")] + [InlineData("SELECT d.X / 3 FROM (SELECT CY AS X FROM T) AS d", "1.0833333333333333333333333333")] + [InlineData("SELECT SUM(DC) / 7 FROM T", "0.6428")] + public void A_column_keeps_its_decimal_kind_through_a_derived_table_and_an_aggregate(string sql, string expected) => + Assert.Equal(expected, Assert.IsType(Query(sql)).ToString(CultureInfo.InvariantCulture)); + + [Fact] + public void Only_the_result_column_is_cut_not_an_argument() => + Assert.Equal(-188.71233644010991d, Assert.IsType(Scalar("PMT(0.05 / 12, 60, 10000)")), 9); +} diff --git a/test/LibRed.Engine.Tests/BitwiseAndDateFunctionTests.cs b/test/LibRed.Engine.Tests/BitwiseAndDateFunctionTests.cs index 49f79e34a..143854830 100644 --- a/test/LibRed.Engine.Tests/BitwiseAndDateFunctionTests.cs +++ b/test/LibRed.Engine.Tests/BitwiseAndDateFunctionTests.cs @@ -58,10 +58,10 @@ public void Bitwise_operators() Assert.Null(Scalar("NULL BAND 3")); // NULL-propagating } - // Bitwise on a byte/short operand promotes to Int32 (as C# does — matching the EF/LINQ contract, not - // ACE's inconsistent narrowing); the value is still correct. + // Bitwise on Integers (16 bits) gives an Integer, as ACE does; a Byte is read as a Long, so anything with a Byte + // gives a Long. [Fact] - public void Bitwise_on_byte_or_short_promotes_to_int() + public void Bitwise_on_integers_gives_an_integer_and_on_bytes_a_long() { string path = Fresh(); try @@ -71,14 +71,13 @@ public void Bitwise_on_byte_or_short_promotes_to_int() e.ExecuteNonQuery("CREATE TABLE Bits (B BYTE, B2 BYTE, S SMALLINT, S2 SMALLINT)"); e.ExecuteNonQuery("INSERT INTO Bits (B, B2, S, S2) VALUES (5, 3, 6, 3)"); - // Mixed narrow+int, and both-operands-narrow (byte&byte, short&short) — all promote to Int32. var r = e.ExecuteQuery("SELECT B BAND 3, BNOT B, S BAND 3, BNOT S, B BAND B2, S BAND S2 FROM Bits").Rows.First(); - Assert.Equal(1, r[0]); Assert.IsType(r[0]); // 5 & 3 - Assert.Equal(-6, r[1]); Assert.IsType(r[1]); // ~5 (promoted, not 250) - Assert.Equal(2, r[2]); Assert.IsType(r[2]); // 6 & 3 - Assert.Equal(-7, r[3]); Assert.IsType(r[3]); // ~6 - Assert.Equal(1, r[4]); Assert.IsType(r[4]); // byte 5 & byte 3 - Assert.Equal(2, r[5]); Assert.IsType(r[5]); // short 6 & short 3 + Assert.Equal(1, r[0]); Assert.IsType(r[0]); // 5 & 3 + Assert.Equal(-6, r[1]); Assert.IsType(r[1]); // ~5 (a Long, not 250) + Assert.Equal(2, r[2]); Assert.IsType(r[2]); // 6 & 3 (3 is a Long) + Assert.Equal((short)-7, r[3]); // ~6 + Assert.Equal(1, r[4]); Assert.IsType(r[4]); // byte 5 & byte 3 + Assert.Equal((short)2, r[5]); // short 6 & short 3 } finally { TemporaryDatabase.Delete(path); } } diff --git a/test/LibRed.Engine.Tests/CalculatedMemoTests.cs b/test/LibRed.Engine.Tests/CalculatedMemoTests.cs new file mode 100644 index 000000000..8809007dd --- /dev/null +++ b/test/LibRed.Engine.Tests/CalculatedMemoTests.cs @@ -0,0 +1,59 @@ +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// A calculated column reads a memo column's text, on insert and on an update that leaves the memo +/// unchanged, not the long-value descriptor the row stores for it. +public class CalculatedMemoTests : TempDatabaseTest +{ + private static QueryEngine Fresh() + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "calculated-memo-"); + var engine = new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); + engine.ExecuteNonQuery( + "CREATE TABLE FullNameBlogs (Id COUNTER NOT NULL, FirstName LONGCHAR NULL, LastName LONGCHAR NULL, " + + "FullName LONGCHAR AS (FirstName + ' ' + LastName), CONSTRAINT PK_FullNameBlogs PRIMARY KEY (Id))"); + return engine; + } + + private static object? FullName(QueryEngine engine) => + engine.ExecuteQuery("SELECT FullName FROM FullNameBlogs").Rows.Single()[0]; + + [Fact] + public void An_insert_reads_the_memo_text() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("INSERT INTO FullNameBlogs (FirstName, LastName) VALUES ('One', 'Unicorn')"); + Assert.Equal("One Unicorn", FullName(engine)); + } + + [Fact] + public void An_update_reads_the_unchanged_memo_text() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("INSERT INTO FullNameBlogs (FirstName, LastName) VALUES ('One', 'Unicorn')"); + engine.ExecuteNonQuery("UPDATE FullNameBlogs SET FirstName = 'Two'"); + Assert.Equal("Two Unicorn", FullName(engine)); + } + + [Fact] + public void A_long_memo_is_read_from_its_page() + { + QueryEngine engine = Fresh(); + string longName = new('x', 300); + engine.ExecuteNonQuery($"INSERT INTO FullNameBlogs (FirstName, LastName) VALUES ('One', '{longName}')"); + engine.ExecuteNonQuery("UPDATE FullNameBlogs SET FirstName = 'Two'"); + Assert.Equal("Two " + longName, FullName(engine)); + } + + // '+' propagates the Null, and a Null text result reads back as empty text, as ACE reads it (page-02e). + [Fact] + public void A_null_memo_propagates_through_plus() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("INSERT INTO FullNameBlogs (FirstName) VALUES ('One')"); + Assert.Equal("", FullName(engine)); + } +} diff --git a/test/LibRed.Engine.Tests/ClusteredConstraintTests.cs b/test/LibRed.Engine.Tests/ClusteredConstraintTests.cs new file mode 100644 index 000000000..eb6553ac3 --- /dev/null +++ b/test/LibRed.Engine.Tests/ClusteredConstraintTests.cs @@ -0,0 +1,115 @@ +using LibRed; +using LibRed.Catalog; +using LibRed.Engine; +using LibRed.Sql.Ast; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// CLUSTERED and NONCLUSTERED after PRIMARY KEY or UNIQUE in a constraint clause — in a +/// CREATE TABLE column or table constraint, and in ALTER TABLE's ADD CONSTRAINT, ADD COLUMN and ALTER COLUMN. ACE +/// accepts every one of those and stores nothing for the word: the file is byte-identical without it, and DAO reports +/// Clustered = False even for an index created with Clustered = True. So LibRed parses the word and +/// drops it, and rejects it wherever ACE does. +/// +public class ClusteredConstraintTests +{ + private static SqlStatement Parse(string sql) => new AntlrSqlParser().ParseStatement(sql); + + [Theory] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY KEY CLUSTERED (Id))", + "CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY KEY (Id))")] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY KEY NONCLUSTERED (Id))", + "CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY KEY (Id))")] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), PRIMARY KEY CLUSTERED (Id))", + "CREATE TABLE T (Id LONG, V TEXT(10), PRIMARY KEY (Id))")] + [InlineData("CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY CLUSTERED, V TEXT(10))", + "CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, V TEXT(10))")] + [InlineData("CREATE TABLE T (Id LONG PRIMARY KEY NONCLUSTERED NOT NULL, V TEXT(10))", + "CREATE TABLE T (Id LONG PRIMARY KEY NOT NULL, V TEXT(10))")] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10) CONSTRAINT uq UNIQUE CLUSTERED)", + "CREATE TABLE T (Id LONG, V TEXT(10) CONSTRAINT uq UNIQUE)")] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT uq UNIQUE NONCLUSTERED (V))", + "CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT uq UNIQUE (V))")] + [InlineData("create table T (Id long, constraint pk primary key clustered (Id))", + "create table T (Id long, constraint pk primary key (Id))")] + [InlineData("ALTER TABLE T ADD CONSTRAINT pk PRIMARY KEY CLUSTERED (Id)", + "ALTER TABLE T ADD CONSTRAINT pk PRIMARY KEY (Id)")] + [InlineData("ALTER TABLE T ADD CONSTRAINT uq UNIQUE NONCLUSTERED (V)", + "ALTER TABLE T ADD CONSTRAINT uq UNIQUE (V)")] + [InlineData("ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY CLUSTERED", + "ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY")] + [InlineData("ALTER TABLE T ALTER COLUMN V TEXT(20) CONSTRAINT uq UNIQUE CLUSTERED", + "ALTER TABLE T ALTER COLUMN V TEXT(20) CONSTRAINT uq UNIQUE")] + public void The_word_parses_to_the_statement_without_it(string withWord, string without) + => Assert.Equal(Describe(Parse(without)), Describe(Parse(withWord))); + + [Theory] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY CLUSTERED KEY (Id))")] + [InlineData("CREATE TABLE T (Id LONG, V TEXT(10), CONSTRAINT pk PRIMARY KEY CLUSTERED CLUSTERED (Id))")] + [InlineData("CREATE TABLE P (Id LONG, PId LONG, CONSTRAINT fk FOREIGN KEY CLUSTERED (PId) REFERENCES Q (Id))")] + [InlineData("CREATE TABLE T (Id LONG CLUSTERED, V TEXT(10))")] + [InlineData("CREATE CLUSTERED INDEX ix ON T (V)")] + [InlineData("CREATE UNIQUE CLUSTERED INDEX ix ON T (V)")] + // Reserved, as ACE reserves both words: unbracketed, neither names a table, column or alias. + [InlineData("CREATE TABLE Clustered (Id LONG)")] + [InlineData("CREATE TABLE T (Clustered LONG)")] + [InlineData("CREATE TABLE T (Nonclustered LONG)")] + [InlineData("SELECT CustomerID AS Clustered FROM Customers")] + public void Ace_rejects_it_everywhere_else_and_so_does_libred(string sql) + => Assert.ThrowsAny(() => Parse(sql)); + + [Fact] + public void A_bracketed_name_is_still_allowed() + { + var create = Assert.IsType(Parse("CREATE TABLE T ([Clustered] LONG, `Nonclustered` LONG)")); + Assert.Equal(["Clustered", "Nonclustered"], create.Columns.Select(c => c.Name)); + } + + [Theory] + [InlineData("CONSTRAINT pk PRIMARY KEY CLUSTERED (Id), CONSTRAINT uq UNIQUE NONCLUSTERED (V)")] + [InlineData("CONSTRAINT pk PRIMARY KEY NONCLUSTERED (Id), CONSTRAINT uq UNIQUE CLUSTERED (V)")] + public void A_table_created_with_the_word_has_the_same_indexes_as_one_without(string constraints) + { + IndexDef[] With(string clause) + { + string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "clustered-"); + try + { + using (var db = JetDatabase.Open(path, readOnly: false)) + new QueryEngine(db).ExecuteNonQuery($"CREATE TABLE T (Id LONG, V TEXT(10), {clause})"); + using (var db = JetDatabase.Open(path)) + return db.Catalog.FindTable("T")!.Indexes.OrderBy(i => i.Name).ToArray(); + } + finally { TemporaryDatabase.Delete(path); } + } + + IndexDef[] plain = With("CONSTRAINT pk PRIMARY KEY (Id), CONSTRAINT uq UNIQUE (V)"); + IndexDef[] clustered = With(constraints); + + Assert.Equal(2, plain.Length); + Assert.Equal( + plain.Select(i => (i.Name, i.IsPrimaryKey, i.IsUnique)), + clustered.Select(i => (i.Name, i.IsPrimaryKey, i.IsUnique))); + } + + /// A statement's parsed shape as text, with its lists spelled out — records compare lists by + /// reference, so two identical parses would otherwise not be equal. + private static string Describe(SqlStatement statement) => statement switch + { + CreateTableStatement c => + $"CREATE {c.Table} cols[{string.Join("; ", c.Columns)}] pk[{string.Join(",", c.PrimaryKey)}] " + + $"pkName={c.PrimaryKeyName} uniques[{string.Join("; ", c.UniqueConstraints.Select(u => $"{u.Name}:{string.Join(",", u.Columns)}"))}] " + + $"fks={c.ForeignKeys.Count} checks={c.CheckConstraints.Count}", + AlterTableStatement a => $"ALTER {a.Table} " + a.Action switch + { + AddPrimaryKeyAction pk => $"pk {pk.Name}:{string.Join(",", pk.Columns)}", + AddUniqueAction uq => $"unique {uq.Unique.Name}:{string.Join(",", uq.Unique.Columns)}", + AddColumnAction add => $"add {add.Column}", + var other => other.ToString(), + }, + _ => statement.ToString(), + }; +} diff --git a/test/LibRed.Engine.Tests/ComparisonOperatorTests.cs b/test/LibRed.Engine.Tests/ComparisonOperatorTests.cs new file mode 100644 index 000000000..5feb51f94 --- /dev/null +++ b/test/LibRed.Engine.Tests/ComparisonOperatorTests.cs @@ -0,0 +1,172 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The comparison operators = <> < > <= >=: how each kind of +/// value compares with another, the truth test against the literal True or False, and where +/// NOT binds. The expected values were measured against ACE, except that a text literal or text column +/// compared with a number reads as a number here, as in SQL Server, where ACE refuses it. +/// +public class ComparisonOperatorTests(ComparisonOperatorTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private const string Guid = "{00112233-4455-6677-8899-AABBCCDDEEFF}"; + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, N LONG, S TEXT(60), NT TEXT(60), D DATETIME, G GUID, B BINARY(4), YN YESNO)", + "INSERT INTO T (Id, N, S, D, YN) VALUES (1, 3, '7', #2020-01-02 12:00:00#, TRUE)", + $"UPDATE T SET G = {Guid}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("compare-ops-", Setup); + + private object? Scalar(string expression) => Query($"SELECT {expression} FROM T"); + + // Text is read as a number in the regional separators, so each query runs under en-US whatever the machine's + // culture. + private object? Query(string sql) => database.Scalar(sql, EnUs); + + [Theory] + [InlineData("CASE WHEN @p = '' THEN 0 ELSE 1 END", 'e', 1)] + [InlineData("@p = 'E'", 'e', true)] + [InlineData("@p < 'f'", 'e', true)] + [InlineData("@p + 'x'", 'e', "ex")] + [InlineData("@p LIKE 'E'", 'e', true)] + [InlineData("@p = 7", '7', true)] + [InlineData("S = @p", '7', true)] + [InlineData("@p * 2", '7', 14.0)] + [InlineData("NOT @p", '0', true)] + public void A_char_parameter_is_one_character_of_text(string expression, char value, object expected) + { + var parameters = new Dictionary { ["p"] = value }; + Assert.Equal(expected, database.Engine.ExecuteQuery($"SELECT {expression} FROM T", parameters).Rows.First()[0]); + } + + [Theory] + [InlineData("2 = TRUE", true)] + [InlineData("N = TRUE", true)] + [InlineData("1.5 = TRUE", true)] + [InlineData("'1' = TRUE", true)] + [InlineData("'abc' = TRUE", true)] + [InlineData("'' = TRUE", true)] + [InlineData("'False' = TRUE", true)] + [InlineData("D = TRUE", true)] + [InlineData("G = TRUE", true)] + [InlineData("TRUE = 'abc'", true)] + [InlineData("0 = TRUE", false)] + [InlineData("'0' = TRUE", false)] + [InlineData("'0' = FALSE", true)] + [InlineData("0.0 = FALSE", true)] + [InlineData("LEFT('0', 1) = FALSE", true)] + [InlineData("'False' = FALSE", false)] + [InlineData("'abc' = FALSE", false)] + [InlineData("FALSE = 2", false)] + [InlineData("1 <> TRUE", false)] + [InlineData("'abc' <> FALSE", true)] + [InlineData("1 = TRUE = TRUE", true)] + [InlineData("(1 = 2) = FALSE", true)] + public void Equality_with_true_or_false_tests_whether_the_value_reads_as_zero(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("1 < TRUE", false)] + [InlineData("TRUE < D", true)] + [InlineData("YN = 1", false)] + [InlineData("YN = -1", true)] + public void Only_equality_with_the_literal_is_a_truth_test(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NULL = TRUE")] + [InlineData("NT = FALSE")] + [InlineData("NT <> 'x'")] + [InlineData("N < NULL")] + public void A_null_side_makes_the_comparison_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("LEFT('10', 2) < 9", false)] + [InlineData("LEFT('10', 2) > 9", true)] + [InlineData("CSTR(10) < 9", false)] + [InlineData("LEFT(' 1 ', 3) = 1", true)] + [InlineData("TRUE < CSTR(5)", true)] + [InlineData("LEFT('12', 2) < TRUE", false)] + [InlineData("'10' > 9", true)] + [InlineData("'1' = 1", true)] + [InlineData("S = 7", true)] + [InlineData("7 = S", true)] + [InlineData("S < 10", true)] + [InlineData("S = 7.0", true)] + public void Text_against_a_number_compares_as_the_number_it_reads_as(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("UCASE('abc') = 1")] + [InlineData("LEFT('abc', 3) = 1")] + [InlineData("'abc' = 1")] + [InlineData("'' = 0")] + [InlineData("'abc' > FALSE")] + [InlineData("D > '2020-01-01'")] + [InlineData("1 = G")] + [InlineData("B < N")] + [InlineData("D = B")] + public void Text_that_is_not_a_number_or_a_guid_or_binary_value_against_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("D = 43832.5", true)] + [InlineData("D > 43832", true)] + [InlineData("40000 < D", true)] + [InlineData("D < N", false)] + [InlineData("D = #2020-01-02 12:00:00#", true)] + public void A_date_against_a_number_compares_as_its_serial(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("G = '" + Guid + "'", true)] + [InlineData("B = 'AB'", true)] + [InlineData("B < 'abc'", true)] + [InlineData("'abc' < B", false)] + [InlineData("G = G", true)] + [InlineData("B = 0x41004200", true)] + [InlineData("B = 0x61004200", false)] + public void A_guid_or_binary_value_against_text_compares_as_text(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'abc' = 'ABC'", true)] + [InlineData("'a ' = 'a'", true)] + [InlineData("' a' = 'a'", false)] + [InlineData("'Z' < 'a'", false)] + [InlineData("'é' < 'f'", true)] + [InlineData("'café' = 'cafe'", false)] + [InlineData("'7.0' = S", false)] + public void Text_against_text_compares_as_text(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NOT 1 = 2", true)] + [InlineData("NOT 1 > 2 = TRUE", true)] + [InlineData("1 < 2 < 3", true)] + [InlineData("3 > 2 > 1", false)] + [InlineData("'b' > 'a' & 'z'", true)] + [InlineData("1 + 1 = 2", true)] + [InlineData("1 & 2 = 12", true)] + public void Comparisons_bind_below_concatenation_and_above_not_left_to_right(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("SELECT COUNT(*) FROM T WHERE S = 7", 1)] + [InlineData("SELECT COUNT(*) FROM T WHERE 2 = TRUE", 1)] + [InlineData("SELECT COUNT(*) FROM T WHERE '0' = TRUE", 0)] + [InlineData("SELECT COUNT(*) FROM T WHERE NOT N = 4", 1)] + [InlineData("SELECT COUNT(*) FROM T WHERE D > 43832", 1)] + public void A_criteria_compares_the_same_way(string sql, int expected) => + Assert.Equal(expected, Query(sql)); +} diff --git a/test/LibRed.Engine.Tests/ConversionFunctionTests.cs b/test/LibRed.Engine.Tests/ConversionFunctionTests.cs index 036e6f9d6..d39c85256 100644 --- a/test/LibRed.Engine.Tests/ConversionFunctionTests.cs +++ b/test/LibRed.Engine.Tests/ConversionFunctionTests.cs @@ -1,87 +1,327 @@ -using LibRed; +using System.Globalization; using LibRed.Engine; using Xunit; namespace LibRed.Engine.Tests; -public class ConversionFunctionTests +/// +/// The conversion functions CBool CByte CInt CLng CSng CDbl CCur +/// CStr CDate CVar: how each reads its argument, rounds and overflows. The expected values were +/// measured against ACE, except that a Null argument gives Null where ACE raises "Invalid use of Null". CLngLng, +/// which ACE does not have, follows CLng. +/// +public class ConversionFunctionTests(ConversionFunctionTests.Database database) + : TempDatabaseTest, IClassFixture { - private static string Fresh() - { - string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "conv-"); - return path; - } - - private static object? Scalar(string expr) - { - string path = Fresh(); - try - { - using var db = JetDatabase.Open(path, readOnly: false); - var e = new QueryEngine(db); - e.ExecuteNonQuery("CREATE TABLE One (Id LONG)"); - e.ExecuteNonQuery("INSERT INTO One (Id) VALUES (1)"); - return e.ExecuteQuery($"SELECT {expr} FROM One").Rows.First()[0]; - } - finally { TemporaryDatabase.Delete(path); } - } + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); - [Fact] - public void Integer_conversions_round_half_to_even() - { - Assert.Equal((short)2, Scalar("CInt(2.5)")); // banker's rounding - Assert.Equal((short)4, Scalar("CInt(3.5)")); - Assert.Equal(-2, Scalar("CLng(-2.5)")); - Assert.Equal((byte)2, Scalar("CByte(2.5)")); - Assert.Equal((byte)255, Scalar("CByte(255)")); - } + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, DC DECIMAL(18,4), TN TEXT(60), NT TEXT(60), DT DATETIME, G GUID, B BINARY(4))", + "INSERT INTO T (Id, DC, TN, DT) VALUES (1, 4.5, '7', #2020-01-02 12:00:00#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("conversion-", Setup); + + // Text is read in the regional format, so each query runs under a fixed culture whatever the machine's. + private object? Scalar(string expression, CultureInfo? culture = null) => + database.Scalar($"SELECT {expression} FROM T", culture ?? EnUs); + + [Theory] + [InlineData("CINT(2.5)", 2)] + [InlineData("CINT(1.5)", 2)] + [InlineData("CINT(-0.5)", 0)] + [InlineData("CINT(-1.5)", -2)] + [InlineData("CINT(TRUE)", -1)] + [InlineData("CINT('2.5')", 2)] + [InlineData("CINT(' 2.5 ')", 2)] + [InlineData("CINT('1,000')", 1000)] + [InlineData("CINT('$5')", 5)] + [InlineData("CINT('&H10')", 16)] + [InlineData("CINT('1e2')", 100)] + [InlineData("CINT('(5)')", -5)] + [InlineData("CINT('5-')", -5)] + [InlineData("CINT(#1899-12-30 06:00#)", 0)] + [InlineData("CINT(TN)", 7)] + public void Cint_reads_text_and_dates_as_numbers_and_rounds_half_to_even(string expression, short expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("CLNG(#2020-01-02#)", 43832)] + [InlineData("CLNG(DT)", 43832)] + [InlineData("CLNG('1.5')", 2)] + [InlineData("CLNG('&H10')", 16)] + [InlineData("CLNG(2147483647.4)", 2147483647)] + public void Clng_converts_to_a_long(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + // CLngLng is VBA's LongLong conversion and a LibRed extension (ACE reports it undefined): CLng's reading, into an + // Int64. Text is read exactly, so the ends of an Int64 survive where a Double would round them. + [Theory] + [InlineData("CLNGLNG(2.5)", 2L)] + [InlineData("CLNGLNG(3.5)", 4L)] + [InlineData("CLNGLNG(-2.5)", -2L)] + [InlineData("CLNGLNG(TRUE)", -1L)] + [InlineData("CLNGLNG(#2020-01-02#)", 43832L)] + [InlineData("CLNGLNG(DC)", 4L)] + [InlineData("CLNGLNG(TN)", 7L)] + [InlineData("CLNGLNG('1.5')", 2L)] + [InlineData("CLNGLNG('1,000')", 1000L)] + [InlineData("CLNGLNG('&H10')", 16L)] + [InlineData("CLNGLNG(1E12)", 1000000000000L)] + [InlineData("CLNGLNG(9223372036854775807)", long.MaxValue)] + [InlineData("CLNGLNG('9223372036854775807')", long.MaxValue)] + [InlineData("CLNGLNG('-9223372036854775808')", long.MinValue)] + public void Clnglng_converts_to_an_int64(string expression, long expected) => + Assert.Equal(expected, Scalar(expression)); [Fact] - public void Float_and_decimal_conversions() - { - Assert.Equal(1.5f, Scalar("CSng(1.5)")); - Assert.Equal(3.0d, Scalar("CDbl(3)")); - Assert.Equal(1.25m, Scalar("CDec(1.25)")); - Assert.Equal(1.5m.ToString(), Scalar("CCur(1.5)")!.ToString()); // currency (existing) - - // A Boolean converts as VARIANT_BOOL, so True is -1 rather than .NET's 1 (verified vs ACE). CByte is - // the exception: a byte cannot hold -1, and ACE reports an overflow, which Convert.ToByte raises. - Assert.Equal((short)-1, Scalar("CInt(True)")); - Assert.Equal(-1, Scalar("CLng(True)")); - Assert.Equal(-1d, Scalar("CDbl(True)")); - Assert.Equal(-1f, Scalar("CSng(True)")); - Assert.Equal(-1m, Scalar("CCur(True)")); - Assert.Throws(() => Scalar("CByte(True)")); - } + public void Clnglng_is_declared_an_int64() => + Assert.Equal(typeof(long), database.Query("SELECT CLNGLNG(Id) FROM T", EnUs).ColumnTypes[0]); + + [Theory] + [InlineData("CBYTE(254.5)", 254)] + [InlineData("CBYTE('2.5')", 2)] + [InlineData("CBYTE('1e2')", 100)] + [InlineData("CBYTE(#1899-12-30 06:00#)", 0)] + public void Cbyte_converts_to_a_byte(string expression, byte expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("CDBL(#2020-01-02#)", 43832)] + [InlineData("CDBL(DT)", 43832.5)] + [InlineData("CDBL(#1899-12-30 06:00#)", 0.25)] + [InlineData("CDBL('$5')", 5)] + [InlineData("CDBL('(5)')", -5)] + [InlineData("CDBL(TRUE)", -1)] + [InlineData("CSNG(DT)", 43832.5)] + [InlineData("CSNG('&H10')", 16)] + [InlineData("CSNG('5-')", -5)] + public void Cdbl_and_csng_read_dates_as_their_serial(string expression, double expected) => + Assert.Equal(expected, Convert.ToDouble(Scalar(expression), CultureInfo.InvariantCulture)); [Fact] - public void String_bool_date_var_conversions() - { - Assert.Equal("1.5", Scalar("CStr(1.5)")); - // "-1", not "True": CStr renders a Boolean as its VARIANT_BOOL number. This is the Jet Expression - // Service's behaviour and differs from the VBA runtime proper — verified vs ACE in - // LibRed.Core.Tests.AceVbaConversionProbeTest. - Assert.Equal("-1", Scalar("CStr(True)")); - // A Double renders at 15 significant digits and a Single at 7 (the OA/VB convention), rather than - // .NET Core's shortest-round-trippable form, which would give "0.30000000000000004" here. - Assert.Equal("0.3", Scalar("CStr(0.1+0.2)")); - Assert.Equal("0.3333333", Scalar("CStr(CSng(1/3))")); - Assert.Equal(false, Scalar("CBool(0)")); - Assert.Equal(true, Scalar("CBool(5)")); - // CBool accepts a numeric string and a non-integral number; Convert.ToBoolean rejects the former. - Assert.Equal(true, Scalar("CBool('-1')")); - Assert.Equal(true, Scalar("CBool(0.5)")); - Assert.Equal(new DateTime(2020, 1, 15), Scalar("CDate('2020-01-15')")); - Assert.Equal(42, Scalar("CVar(42)")); // passthrough - } + public void Csng_keeps_single_precision() => + Assert.Equal(1f / 3, Scalar("CSNG(1/3)")); + + [Theory] + [InlineData("CCUR(DT)", "43832.5")] + [InlineData("CCUR('1e2')", "100")] + [InlineData("CCUR('(5)')", "-5")] + [InlineData("CCUR(1.23456)", "1.2346")] + [InlineData("CCUR(TRUE)", "-1")] + // A number written with a decimal point, and text, are read exactly rather than as a Double (verified vs ACE). + [InlineData("CCUR('12345678901234.5678')", "12345678901234.5678")] + [InlineData("CCUR(12345678901234.5678)", "12345678901234.5678")] + [InlineData("CCUR('922337203685477.5807')", "922337203685477.5807")] + [InlineData("CCUR('1234567890123.45678')", "1234567890123.4568")] + public void Ccur_converts_to_four_places(string expression, string expected) => + Assert.Equal(decimal.Parse(expected, CultureInfo.InvariantCulture), Scalar(expression)); + + // CDec is a LibRed extension (ACE's expression service refuses it), reading its argument as CCur does. + [Theory] + [InlineData("CDEC(12345678901234567.123456789)", "12345678901234567.123456789")] + [InlineData("CDEC(-12345678901234567.123456789)", "-12345678901234567.123456789")] + [InlineData("CDEC('12345678901234567.123456789')", "12345678901234567.123456789")] + [InlineData("CDEC('1e3')", "1000")] + [InlineData("CDEC('&HFF')", "255")] + [InlineData("CDEC(0.1) + CDEC(0.2)", "0.3")] + public void Cdec_keeps_every_written_place(string expression, string expected) => + Assert.Equal(decimal.Parse(expected, CultureInfo.InvariantCulture), Scalar(expression)); + + [Theory] + [InlineData("CBOOL(#2020-01-02#)", true)] + [InlineData("CBOOL(DT)", true)] + [InlineData("CBOOL(#1899-12-30 06:00#)", true)] + [InlineData("CBOOL('$5')", true)] + [InlineData("CBOOL('&H10')", true)] + [InlineData("CBOOL('True')", true)] + [InlineData("CBOOL('False')", false)] + [InlineData("CBOOL('0')", false)] + [InlineData("CBOOL('-0')", false)] + [InlineData("CBOOL(0.5)", true)] + [InlineData("CBOOL(0)", false)] + public void Cbool_is_true_for_anything_that_reads_as_non_zero(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("CSTR(TRUE)", "-1")] + [InlineData("CSTR(DC)", "4.5")] + [InlineData("CSTR(1/3)", "0.333333333333333")] + [InlineData("CSTR(CSNG(1/3))", "0.3333333")] + [InlineData("CSTR(1E-5)", "0.00001")] + [InlineData("CSTR(1E300)", "1E+300")] + [InlineData("CSTR(CCUR(1.23456))", "1.2346")] + [InlineData("CSTR(#2020-01-02#)", "1/2/2020")] + [InlineData("CSTR(DT)", "1/2/2020 12:00:00 PM")] + [InlineData("CSTR(#1899-12-30 06:00#)", "6:00:00 AM")] + [InlineData("CSTR(G)", "{00112233-4455-6677-8899-AABBCCDDEEFF}")] + [InlineData("CSTR(B)", "AB")] + public void Cstr_writes_a_value_as_ampersand_does(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("CDATE('1')", "1899-12-31 00:00:00")] + [InlineData("CDATE(' 1 ')", "1899-12-31 00:00:00")] + [InlineData("CDATE('0')", "1899-12-30 00:00:00")] + [InlineData("CDATE('$5')", "1900-01-04 00:00:00")] + [InlineData("CDATE('&H10')", "1900-01-15 00:00:00")] + [InlineData("CDATE('1e2')", "1900-04-09 00:00:00")] + [InlineData("CDATE('(5)')", "1899-12-25 00:00:00")] + [InlineData("CDATE(TN)", "1900-01-06 00:00:00")] + [InlineData("CDATE(TRUE)", "1899-12-29 00:00:00")] + [InlineData("CDATE(43832.5)", "2020-01-02 12:00:00")] + [InlineData("CDATE('12:00')", "1899-12-30 12:00:00")] + [InlineData("CDATE('1/2/2020')", "2020-01-02 00:00:00")] + [InlineData("CDATE('2020-01-02 13:30')", "2020-01-02 13:30:00")] + [InlineData("CDATE('Jan 2, 2020')", "2020-01-02 00:00:00")] + [InlineData("CDATE(DT)", "2020-01-02 12:00:00")] + public void Cdate_reads_a_number_as_a_serial_and_other_text_as_a_date(string expression, string expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)).ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + + // Measured against ACE under en-AU (day before month). {year} is the current year. + [Theory] + [InlineData("2.5", "1899-12-30 02:05:00")] + [InlineData("2.5.20", "1899-12-30 02:05:20")] + [InlineData("0.5", "1899-12-30 00:05:00")] + [InlineData("10.30.45", "1899-12-30 10:30:45")] + [InlineData("1 .5", "1899-12-30 01:05:00")] + [InlineData("13.30", "1899-12-30 13:30:00")] + [InlineData("1.30 PM", "1899-12-30 13:30:00")] + [InlineData("2.5 PM", "1899-12-30 14:05:00")] + [InlineData("1:2:3", "1899-12-30 01:02:03")] + [InlineData("13:00 PM", "1899-12-30 13:00:00")] + [InlineData("12 AM", "1899-12-30 00:00:00")] + [InlineData("3PM", "1899-12-30 15:00:00")] + [InlineData("25.5", "1900-01-24 12:00:00")] + [InlineData("2.60", "1900-01-01 14:24:00")] + [InlineData("31.12", "1900-01-30 02:52:48")] + [InlineData(".5", "1899-12-30 12:00:00")] + [InlineData("1.", "1899-12-31 00:00:00")] + [InlineData("1-", "1899-12-29 00:00:00")] + [InlineData("2020", "1905-07-12 00:00:00")] + [InlineData("1,000", "2000-01-01 00:00:00")] + [InlineData("1,2020", "2020-01-01 00:00:00")] + [InlineData("1,000.5", "1902-09-26 12:00:00")] + [InlineData("1.000,5", "1899-12-31 00:00:43")] + [InlineData("1,5", "{year}-05-01 00:00:00")] + [InlineData("1 2", "{year}-02-01 00:00:00")] + [InlineData("13/2", "{year}-02-13 00:00:00")] + [InlineData("2/13", "{year}-02-13 00:00:00")] + [InlineData("1/2 3:04", "{year}-02-01 03:04:00")] + [InlineData("1-2-2020", "2020-02-01 00:00:00")] + [InlineData("1 / 2 / 2020", "2020-02-01 00:00:00")] + [InlineData("2020-1-2", "2020-01-02 00:00:00")] + [InlineData("2020 1 2", "2020-01-02 00:00:00")] + [InlineData("1/2/29", "2029-02-01 00:00:00")] + [InlineData("1/2/30", "2030-02-01 00:00:00")] + [InlineData("1/2/99", "1999-02-01 00:00:00")] + [InlineData("1/2/0", "2000-02-01 00:00:00")] + [InlineData("1/2/100", "0100-02-01 00:00:00")] + [InlineData("12:00 1/2/2020", "2020-02-01 12:00:00")] + [InlineData("1/2/2020 1:30 PM", "2020-02-01 13:30:00")] + [InlineData("Jan 2", "{year}-01-02 00:00:00")] + [InlineData("Jan 2020", "2020-01-01 00:00:00")] + [InlineData("Feb 30", "2030-02-01 00:00:00")] + [InlineData("Feb 29 2020", "2020-02-29 00:00:00")] + [InlineData("2 January 2020", "2020-01-02 00:00:00")] + [InlineData("January 2, 2020", "2020-01-02 00:00:00")] + [InlineData("2020-01-02 12:00:00", "2020-01-02 12:00:00")] + public void Cdate_reads_text_as_ole_automation_does(string text, string expected) => + Assert.Equal( + expected.Replace("{year}", DateTime.Today.Year.ToString(CultureInfo.InvariantCulture)), + Assert.IsType(Scalar($"CDATE('{text}')", CultureInfo.GetCultureInfo("en-AU"))) + .ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("2.5.2020")] + [InlineData("31/2/2020")] + [InlineData("Jan")] + [InlineData("tues")] + [InlineData("1st Jan")] + [InlineData("2020-01-02T12:00")] + [InlineData("2020-13-01")] + [InlineData("1.2.3.4")] + [InlineData("24:00")] + [InlineData("1:60")] + [InlineData("1//2")] + [InlineData("1/2/2020/")] + [InlineData("1/2/10000")] + [InlineData("Feb 29 2019")] + [InlineData("noon")] + public void Cdate_of_text_that_is_neither_a_date_nor_a_number_is_a_type_mismatch(string text) => + Assert.Throws(() => Scalar($"CDATE('{text}')", CultureInfo.GetCultureInfo("en-AU"))); [Fact] - public void Conversions_propagate_null() - { - Assert.Null(Scalar("CInt(NULL)")); - Assert.Null(Scalar("CDbl(NULL)")); - Assert.Null(Scalar("CStr(NULL)")); - Assert.Null(Scalar("CDate(NULL)")); - Assert.Null(Scalar("CVar(NULL)")); - } + public void Cdate_keeps_a_fraction_of_a_second() => + Assert.Equal(1E-5, Scalar("CDBL(CDATE(1E-5))")); + + // A LibRed extension, which LibRed's own SQL relies on for a time with milliseconds; ACE refuses the fourth part. + [Theory] + [InlineData("TIMEVALUE('12:30:45.123')", "1899-12-30 12:30:45.1230000", "en-US")] + [InlineData("TIMEVALUE('12:30:45.5')", "1899-12-30 12:30:45.5000000", "en-US")] + [InlineData("TIMEVALUE('12:30:45.1234567')", "1899-12-30 12:30:45.1234567", "en-US")] + [InlineData("CDATE('2020-01-02 12:30:45.007')", "2020-01-02 12:30:45.0070000", "en-US")] + [InlineData("TIMEVALUE('12:30:45.123')", "1899-12-30 12:30:45.1230000", "de-DE")] + public void Time_text_may_have_a_fraction_of_a_second(string expression, string expected, string culture) => + Assert.Equal(expected, Assert.IsType(Scalar(expression, CultureInfo.GetCultureInfo(culture))) + .ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("TIMEVALUE('12.30.45.5')")] + [InlineData("TIMEVALUE('12:30:45.12345678')")] + [InlineData("TIMEVALUE('12:30:45.1.2')")] + public void Only_seconds_take_a_short_fraction(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("CVAR(1)", 1)] + [InlineData("CVAR('abc')", "abc")] + [InlineData("CVAR(TRUE)", true)] + public void Cvar_passes_its_argument_through(string expression, object expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("CBOOL(NULL)")] + [InlineData("CBYTE(NULL)")] + [InlineData("CINT(NT)")] + [InlineData("CLNG(NULL)")] + [InlineData("CLNGLNG(NULL)")] + [InlineData("CSNG(NULL)")] + [InlineData("CDBL(NULL)")] + [InlineData("CCUR(NT)")] + [InlineData("CSTR(NULL)")] + [InlineData("CDATE(NT)")] + [InlineData("CVAR(NULL)")] + public void Null_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("CBYTE(256)")] + [InlineData("CBYTE(255.5)")] + [InlineData("CBYTE(-1)")] + [InlineData("CBYTE(TRUE)")] + [InlineData("CINT(32767.5)")] + [InlineData("CLNG(2147483647.5)")] + [InlineData("CLNGLNG('9223372036854775808')")] + [InlineData("CLNGLNG(1E19)")] + [InlineData("CSNG(1E300)")] + [InlineData("CCUR(1234567890123456)")] + [InlineData("CDATE(1E300)")] + public void A_value_past_the_type_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("CBOOL('yes')")] + [InlineData("CINT('abc')")] + [InlineData("CDBL('')")] + [InlineData("CLNG(G)")] + [InlineData("CLNGLNG(G)")] + [InlineData("CLNGLNG('abc')")] + [InlineData("CDATE('abc')")] + [InlineData("CDATE(B)")] + public void A_value_that_does_not_convert_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); } diff --git a/test/LibRed.Engine.Tests/CurrencyArithmeticTests.cs b/test/LibRed.Engine.Tests/CurrencyArithmeticTests.cs new file mode 100644 index 000000000..7470450cb --- /dev/null +++ b/test/LibRed.Engine.Tests/CurrencyArithmeticTests.cs @@ -0,0 +1,65 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// A Currency result is a Currency at every step: four places, half to even, and past ±922,337,203,685,477.5807 an +/// overflow — as CCur works it (verified vs ACE). What counts as a Currency result is ACE's: Currency with Currency or a +/// whole number; not Currency with a written decimal, a Double, or a whole number too big for a Long, which ACE reads +/// as a Decimal. +/// +public class CurrencyArithmeticTests(CurrencyArithmeticTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Z is the largest whole Currency. + private static readonly string[] Setup = + [ + "CREATE TABLE C (K LONG, A CURRENCY, B CURRENCY, Z CURRENCY, Y CURRENCY, I LONG, D DOUBLE, L BIGINT)", + "INSERT INTO C (K, A, B, Z, Y, I, D, L) VALUES (1, 1.2345, 0.0003, 922337203685477, 1000000000.5, 3, 1.5, 1000000000000)", + ]; + + public sealed class Database() : SharedDatabase("currency-arith-", Setup); + + private object? Scalar(string expression) => + database.Scalar($"SELECT {expression} FROM C", CultureInfo.InvariantCulture); + + [Theory] + [InlineData("A * A", "1.524")] + [InlineData("A * B", "0.0004")] + [InlineData("(A * A) * 1000", "1524")] // the product is rounded before it is multiplied + [InlineData("A * A + B", "1.5243")] + [InlineData("A * I", "3.7035")] + [InlineData("A - B", "1.2342")] + public void A_currency_result_keeps_four_places(string expression, string expected) => + Assert.Equal(decimal.Parse(expected, CultureInfo.InvariantCulture), Convert.ToDecimal(Scalar(expression))); + + [Theory] + [InlineData("A + 0.00005", "1.23455")] // with a written decimal it is a Decimal + [InlineData("Z + 0.5808", "922337203685477.5808")] // ... so past the Currency range is no overflow + [InlineData("Y * 864000000000", "864000000432000000000")] // a literal past a Long is a Decimal too + [InlineData("Z + 1000000000000", "923337203685477")] + public void A_currency_with_a_decimal_is_not_a_currency(string expression, string expected) => + Assert.Equal(decimal.Parse(expected, CultureInfo.InvariantCulture), Convert.ToDecimal(Scalar(expression))); + + [Theory] + [InlineData("Z + 1")] + [InlineData("-Z - 1")] + [InlineData("Z * I")] + [InlineData("Z + Z")] + [InlineData("Y * Y")] + [InlineData("Y * L")] + [InlineData("L * Y")] + public void Past_the_currency_range_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + // The literal's type decides the places as well: ACE takes 864000000000 / 7 to a Decimal of none. + [Fact] + public void A_literal_past_a_long_has_no_places() => + Assert.Equal(123428571428m, Convert.ToDecimal(Scalar("864000000000 / 7"))); + + // Its value is still the Int64 LibRed reads it as, so MOD and \ stay exact over it. + [Fact] + public void A_literal_past_a_long_is_still_an_int64() => + Assert.Equal(136000000000L, Scalar("L MOD 864000000000")); +} diff --git a/test/LibRed.Engine.Tests/DataLengthTests.cs b/test/LibRed.Engine.Tests/DataLengthTests.cs new file mode 100644 index 000000000..e396a54a3 --- /dev/null +++ b/test/LibRed.Engine.Tests/DataLengthTests.cs @@ -0,0 +1,74 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// SQL Server's DATALENGTH: the bytes a value takes as Access stores it. Access has no such function; this is +/// a LibRed extension. +/// +public class DataLengthTests(DataLengthTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE D (Id LONG, BT BYTE, SI SHORT, LG LONG, BI BIGINT, SG REAL, DB FLOAT, CY CURRENCY, " + + "DC DECIMAL(18,4), TX TEXT(20), MM MEMO, DT DATETIME, YN YESNO, GD GUID, BN BINARY(10), VB VARBINARY(10))", + "INSERT INTO D (Id, BT, SI, LG, BI, SG, DB, CY, DC, TX, MM, DT, YN, GD, BN, VB) VALUES (1, 1, 2, 3, 4, 1.5, 2.5, 3.25, " + + "4.5, 'abc ', 'hello', #2020-01-02#, TRUE, {00112233-4455-6677-8899-AABBCCDDEEFF}, 0x010203, 0x010203)", + "INSERT INTO D (Id, TX) VALUES (2, '')", + ]; + + public sealed class Database() : SharedDatabase("datalength-", Setup); + + private object? Scalar(string expression, int id = 1) => + database.Scalar($"SELECT {expression} FROM D WHERE Id = {id}", CultureInfo.InvariantCulture); + + [Theory] + [InlineData("BT", 1)] + [InlineData("SI", 2)] + [InlineData("LG", 4)] + [InlineData("BI", 8)] + [InlineData("SG", 4)] + [InlineData("DB", 8)] + [InlineData("CY", 8)] + [InlineData("DC", 17)] + [InlineData("DT", 8)] + [InlineData("YN", 1)] + [InlineData("GD", 16)] + public void A_value_takes_the_size_its_type_is_stored_in(string column, int expected) => + Assert.Equal(expected, Scalar($"DATALENGTH({column})")); + + [Theory] + [InlineData("TX", 8)] // 'abc ': the trailing space counts + [InlineData("MM", 10)] + [InlineData("'héllo'", 10)] + [InlineData("ChrW(12288)", 2)] + [InlineData("VB", 3)] // variable binary: its own length + [InlineData("BN", 10)] // fixed binary: stored padded to the column's width + [InlineData("0x010203", 3)] + public void Text_is_two_bytes_a_character_and_binary_its_length(string expression, int expected) => + Assert.Equal(expected, Scalar($"DATALENGTH({expression})")); + + [Fact] + public void Empty_text_is_zero_bytes() => Assert.Equal(0, Scalar("DATALENGTH(TX)", id: 2)); + + [Theory] + [InlineData("1", 4)] + [InlineData("CCUR(1)", 8)] + [InlineData("CDEC(1)", 17)] + [InlineData("LG + 1", 4)] + public void An_expression_has_its_result_types_size(string expression, int expected) => + Assert.Equal(expected, Scalar($"DATALENGTH({expression})")); + + [Fact] + public void Null_has_no_length() + { + Assert.Null(Scalar("DATALENGTH(NULL)")); + Assert.Null(Scalar("DATALENGTH(BT)", id: 2)); + } + + [Fact] + public void The_column_is_a_long() => + Assert.Equal(typeof(int), database.Query("SELECT DATALENGTH(TX) FROM D", CultureInfo.InvariantCulture).ColumnTypes[0]); +} diff --git a/test/LibRed.Engine.Tests/DateDiffMillisecondTests.cs b/test/LibRed.Engine.Tests/DateDiffMillisecondTests.cs index 31474ee2c..ae145fa77 100644 --- a/test/LibRed.Engine.Tests/DateDiffMillisecondTests.cs +++ b/test/LibRed.Engine.Tests/DateDiffMillisecondTests.cs @@ -55,10 +55,24 @@ public void Returns_a_long_not_an_int() public void Other_intervals_still_return_int(string interval) => Assert.IsType(Eval(Fresh(), $"DATEDIFF('{interval}', #2020-01-01 00:00:00#, #2021-03-04 05:06:07#)")); + // The column is declared as the values are — what a reader's GetFieldType reports, and what EF reads with. + [Theory] + [InlineData("ms", typeof(long))] + [InlineData("MS", typeof(long))] + [InlineData("s", typeof(int))] + [InlineData("d", typeof(int))] + public void The_column_is_declared_as_its_values(string interval, Type expected) + { + var result = Fresh().ExecuteQuery( + $"SELECT DATEDIFF('{interval}', #2020-01-01 00:00:00#, #2020-01-02 00:00:00#) FROM `Shippers` WHERE `ShipperID` = 1"); + Assert.Equal(expected, result.ColumnTypes[0]); + Assert.IsType(expected, result.Rows.Single()[0]); + } + // Only the abbreviation is accepted, matching DatePart and the rest of the interval table. The full word // is what EF used to emit and what the Jet translators now no longer send. [Fact] public void The_full_word_is_not_an_interval() - => Assert.Throws( + => Assert.Throws( () => Eval(Fresh(), "DATEDIFF('millisecond', #2020-01-01 00:00:00#, #2020-01-01 00:00:01#)")); } diff --git a/test/LibRed.Engine.Tests/DateFunctionTests.cs b/test/LibRed.Engine.Tests/DateFunctionTests.cs new file mode 100644 index 000000000..01ddd9124 --- /dev/null +++ b/test/LibRed.Engine.Tests/DateFunctionTests.cs @@ -0,0 +1,292 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The date functions Year Month Day Hour Minute Second Weekday +/// DatePart DateAdd DateDiff DateSerial TimeSerial MonthName +/// WeekdayName DateValue TimeValue IsDate. The expected values were measured against ACE +/// under en-AU (day before month, weeks starting on Monday), except that a Null argument gives Null where ACE raises +/// an error. +/// +public class DateFunctionTests(DateFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly CultureInfo EnAu = CultureInfo.GetCultureInfo("en-AU"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, NT TEXT(60), DT DATETIME)", + "INSERT INTO T (Id, DT) VALUES (1, #2020-01-02 12:00:00#)", + ]; + + public sealed class Database() : SharedDatabase("date-", Setup); + + // Text and the system week settings follow the regional format, so each query runs under a fixed culture. + private object? Scalar(string expression, CultureInfo? culture = null) => + database.Scalar($"SELECT {expression} FROM T", culture ?? EnAu); + + private static string? Text(object? value) => value switch + { + null => null, + DateTime date => date.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture), + IFormattable number => number.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString(), + }; + + [Theory] + [InlineData("MONTH('2/01/2020')", "1")] + [InlineData("DAY('2/01/2020')", "2")] + [InlineData("WEEKDAY('2/01/2020')", "5")] + [InlineData("YEAR(43832.5)", "2020")] + [InlineData("HOUR(43832.5)", "12")] + [InlineData("WEEKDAY(43832.5)", "5")] + [InlineData("YEAR('43832.5')", "2020")] + [InlineData("DAY('43832.5')", "2")] + [InlineData("YEAR(TRUE)", "1899")] + [InlineData("DAY(TRUE)", "29")] + [InlineData("WEEKDAY(TRUE)", "6")] + [InlineData("DAY(1)", "31")] + [InlineData("WEEKDAY(1)", "1")] + [InlineData("DAY(-1.25)", "29")] + [InlineData("HOUR(-1.25)", "6")] + [InlineData("YEAR('$5')", "1900")] + [InlineData("DAY('$5')", "4")] + [InlineData("YEAR('12:30')", "1899")] + [InlineData("DAY('12:30')", "30")] + [InlineData("WEEKDAY('12:30')", "7")] + [InlineData("HOUR(CCUR(43832.25))", "6")] + [InlineData("YEAR('2.5')", "1899")] + [InlineData("HOUR('2.5')", "2")] + [InlineData("MINUTE('2.5')", "5")] + [InlineData("HOUR(1.75)", "18")] + [InlineData("HOUR('1.75')", "18")] + [InlineData("DAY('2020')", "12")] + [InlineData("MONTH('1,000')", "1")] + public void Date_parts_read_text_as_a_date_and_numbers_as_a_serial(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression))); + + [Theory] + [InlineData("DATEPART('yyyy', 43832.75)", "2020")] + [InlineData("DATEPART('m', '2/01/2020')", "1")] + [InlineData("DATEPART('y', '2/01/2020')", "2")] + [InlineData("DATEPART('w', '2/01/2020')", "5")] + [InlineData("DATEPART('ww', '2/01/2020')", "1")] + [InlineData("DATEPART('h', 43832.75)", "18")] + [InlineData("WEEKDAY(#2020-01-02#, 0)", "4")] + [InlineData("WEEKDAY(#2020-01-02#, 2)", "4")] + [InlineData("WEEKDAY(#2020-01-02#, 4)", "2")] + [InlineData("WEEKDAY(#2020-01-02#, 7)", "6")] + [InlineData("WEEKDAY(DT, '2')", "4")] + [InlineData("WEEKDAY(DT, 1.5)", "4")] + [InlineData("DATEPART('w', #2020-01-02#, 0)", "4")] + [InlineData("DATEPART('ww', #2021-01-03#, 0)", "1")] + [InlineData("DATEPART('ww', #2021-01-03#, 1)", "2")] + [InlineData("DATEPART('ww', #2021-01-03#, 2)", "1")] + [InlineData("DATEPART('ww', #2021-01-03#, 7)", "2")] + public void Datepart_and_weekday_count_from_the_first_day_of_the_week(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression))); + + [Theory] + [InlineData("#2021-01-01#, 1, 1", 1)] + [InlineData("#2020-12-31#, 1, 0", 53)] + [InlineData("#2021-01-01#, 0, 2", 53)] + [InlineData("#2019-12-30#, 0, 2", 1)] + [InlineData("#2024-12-30#, 0, 2", 1)] + [InlineData("#2021-01-01#, 0, 3", 52)] + [InlineData("#2020-12-31#, 0, 3", 52)] + [InlineData("#2019-12-30#, 0, 3", 52)] + [InlineData("#2024-12-30#, 2, 3", 53)] + [InlineData("#2021-01-01#, 1, 2", 53)] + [InlineData("#2019-12-30#, 1, 2", 1)] + [InlineData("#2020-12-31#, 1, 2", 53)] + [InlineData("#2021-01-01#, 1, 3", 52)] + [InlineData("#2024-12-30#, 1, 3", 52)] + [InlineData("#2020-12-31#, 4, 2", 1)] + [InlineData("#2019-12-30#, 4, 2", 52)] + [InlineData("#2021-01-01#, 4, 2", 1)] + [InlineData("#2021-01-01#, 4, 3", 53)] + [InlineData("#2021-01-01#, 7, 2", 52)] + [InlineData("#2019-12-30#, 7, 2", 53)] + [InlineData("#2024-12-30#, 7, 3", 52)] + [InlineData("#2003-12-29#, 2, 2", 1)] + [InlineData("#2003-12-28#, 1, 2", 53)] + public void Week_of_year_follows_the_first_week_rule(string arguments, int expected) => + Assert.Equal(expected, Scalar($"DATEPART('ww', {arguments})")); + + [Theory] + [InlineData("DATEADD('yyyy', '2', '2/01/2020')", "2022-01-02 00:00:00")] + [InlineData("DATEADD('q', '2', '2/01/2020')", "2020-07-02 00:00:00")] + [InlineData("DATEADD('y', '2', '2/01/2020')", "2020-01-04 00:00:00")] + [InlineData("DATEADD('ww', '2', '2/01/2020')", "2020-01-16 00:00:00")] + [InlineData("DATEADD('h', '2', '2/01/2020')", "2020-01-02 02:00:00")] + [InlineData("DATEADD('d', TRUE, DT)", "2020-01-01 12:00:00")] + [InlineData("DATEADD('d', 1, 43832)", "2020-01-03 00:00:00")] + [InlineData("DATEADD('m', 1, #2020-01-30#)", "2020-02-29 00:00:00")] + [InlineData("DATEADD('yyyy', 1, #2020-02-29#)", "2021-02-28 00:00:00")] + [InlineData("DATEADD('m', 15, #2020-01-31#)", "2021-04-30 00:00:00")] + [InlineData("DATEADD('m', -15, #2020-01-31#)", "2018-10-31 00:00:00")] + [InlineData("DATEADD('s', 90, #2020-01-01#)", "2020-01-01 00:01:30")] + [InlineData("DATEADD('s', -90, #2020-01-01#)", "2019-12-31 23:58:30")] + [InlineData("DATEADD('d', 2.5, #2020-01-01#)", "2020-01-03 00:00:00")] + [InlineData("DATEADD('d', 0.5, DT)", "2020-01-02 12:00:00")] + [InlineData("DATEADD('h', -30, #1899-12-31 06:00#)", "1899-12-30 00:00:00")] + [InlineData("DATEADD('s', 1, #1899-12-29 23:59:59#)", "1899-12-30 00:00:00")] + [InlineData("DATEADD('yyyy', -1920, DT)", "0100-01-02 12:00:00")] + [InlineData("DATEADD('yyyy', 7979, DT)", "9999-01-02 12:00:00")] + public void Dateadd_truncates_the_number(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression))); + + [Theory] + [InlineData("DATEDIFF('yyyy', 43832.25, '2/02/2020')", 0)] + [InlineData("DATEDIFF('m', 43832.25, '2/02/2020')", 1)] + [InlineData("DATEDIFF('d', 43832.25, '2/02/2020')", 31)] + [InlineData("DATEDIFF('w', 43832.25, '2/02/2020')", 4)] + [InlineData("DATEDIFF('ww', 43832.25, '2/02/2020')", 5)] + [InlineData("DATEDIFF('h', 43832.25, '2/02/2020')", 738)] + [InlineData("DATEDIFF('n', 43832.25, '2/02/2020')", 44280)] + [InlineData("DATEDIFF('s', 43832.25, '2/02/2020')", 2656800)] + [InlineData("DATEDIFF('w', #2020-12-31 23:59:59#, #2021-01-01 00:00:00#)", 0)] + [InlineData("DATEDIFF('w', #2020-01-04#, #2020-01-19#)", 2)] + [InlineData("DATEDIFF('w', #2020-01-05#, #2020-01-18#)", 1)] + [InlineData("DATEDIFF('w', #2020-01-19#, #2020-01-04#)", -2)] + [InlineData("DATEDIFF('w', #1899-12-29 18:00#, #1899-12-30 06:00#)", 0)] + [InlineData("DATEDIFF('ww', #2020-01-04#, #2020-01-19#)", 3)] + [InlineData("DATEDIFF('ww', #2020-01-19#, #2020-01-04#)", -3)] + [InlineData("DATEDIFF('h', #2020-12-31 23:59:59#, #2021-01-01 00:00:00#)", 1)] + [InlineData("DATEDIFF('h', #2020-01-01 10:59:59#, #2020-01-01 11:00:01#)", 1)] + [InlineData("DATEDIFF('h', #2020-01-01 11:00:59#, #2020-01-01 10:59:00#)", -1)] + [InlineData("DATEDIFF('n', #2020-12-31 23:59:59#, #2021-01-01 00:00:00#)", 1)] + [InlineData("DATEDIFF('n', #2020-01-01 10:59:59#, #2020-01-01 11:00:01#)", 1)] + [InlineData("DATEDIFF('d', #1899-12-29 18:00#, #1899-12-30 06:00#)", 1)] + [InlineData("DATEDIFF('d', DT, DT + 0.99)", 1)] + [InlineData("DATEDIFF('w', #2020-01-01#, #2020-02-01#, 8)", 4)] + [InlineData("DATEDIFF('ww', #2020-01-01#, #2020-02-01#, 7)", 5)] + [InlineData("DATEDIFF('ww', #2020-01-01#, #2020-02-01#, 0)", 4)] + [InlineData("DATEDIFF('d', #2020-01-01#, #2020-01-02#, 1, 4)", 1)] + public void Datediff_counts_boundaries(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("DATESERIAL(29, 1, 1)", "2029-01-01 00:00:00")] + [InlineData("DATESERIAL(49, 1, 1)", "2049-01-01 00:00:00")] + [InlineData("DATESERIAL(50, 1, 1)", "1950-01-01 00:00:00")] + [InlineData("DATESERIAL(99, 1, 1)", "1999-01-01 00:00:00")] + [InlineData("DATESERIAL(0, 1, 1)", "2000-01-01 00:00:00")] + [InlineData("DATESERIAL(-1, 1, 1)", "1999-01-01 00:00:00")] + [InlineData("DATESERIAL(99, 12, 32)", "2000-01-01 00:00:00")] + [InlineData("DATESERIAL(100, 0, 1)", "1999-12-01 00:00:00")] + [InlineData("DATESERIAL(100, 1, 1)", "0100-01-01 00:00:00")] + [InlineData("DATESERIAL(2020, 0, 0)", "2019-11-30 00:00:00")] + [InlineData("DATESERIAL(2020, -1, -1)", "2019-10-30 00:00:00")] + [InlineData("DATESERIAL(2020, 1, 32767)", "2109-09-17 00:00:00")] + [InlineData("DATESERIAL(2020, 1.5, 1.5)", "2020-02-02 00:00:00")] + [InlineData("DATESERIAL(2020, 2.5, 2.5)", "2020-02-02 00:00:00")] + [InlineData("DATESERIAL('2020', '1', '2')", "2020-01-02 00:00:00")] + [InlineData("DATESERIAL(2020, TRUE, 1)", "2019-11-01 00:00:00")] + [InlineData("DATESERIAL(2020, 15, 1)", "2021-03-01 00:00:00")] + [InlineData("DATESERIAL(2020, -15, 1)", "2018-09-01 00:00:00")] + [InlineData("DATESERIAL(2020, 1, -40)", "2019-11-21 00:00:00")] + [InlineData("TIMESERIAL(0, 0, 90)", "1899-12-30 00:01:30")] + [InlineData("TIMESERIAL(0, 0, -90)", "1899-12-30 00:01:30")] + [InlineData("TIMESERIAL(0, -90, 0)", "1899-12-30 01:30:00")] + [InlineData("TIMESERIAL(10, 0, -90)", "1899-12-30 09:58:30")] + [InlineData("TIMESERIAL(-1, 0, 0)", "1899-12-30 01:00:00")] + [InlineData("TIMESERIAL(0, 0, -1)", "1899-12-30 00:00:01")] + [InlineData("TIMESERIAL(-12, 0, 0)", "1899-12-30 12:00:00")] + [InlineData("TIMESERIAL(-24, 0, 0)", "1899-12-29 00:00:00")] + [InlineData("TIMESERIAL(-25, 0, 0)", "1899-12-29 01:00:00")] + [InlineData("TIMESERIAL(25, 0, 0)", "1899-12-31 01:00:00")] + [InlineData("TIMESERIAL(0, 0, 32767)", "1899-12-30 09:06:07")] + [InlineData("TIMESERIAL(1.5, 1.5, 1.5)", "1899-12-30 02:02:02")] + [InlineData("TIMESERIAL('1', '2', '3')", "1899-12-30 01:02:03")] + public void Dateserial_and_timeserial_carry_out_of_range_parts(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression))); + + [Theory] + [InlineData("MONTHNAME(1.5)", "February")] + [InlineData("MONTHNAME('2')", "February")] + [InlineData("MONTHNAME(12, 1)", "Dec")] + [InlineData("WEEKDAYNAME(1)", "Monday")] + [InlineData("WEEKDAYNAME(7, TRUE)", "Sun")] + [InlineData("WEEKDAYNAME(1.5)", "Tuesday")] + [InlineData("WEEKDAYNAME('3')", "Wednesday")] + [InlineData("WEEKDAYNAME(1, FALSE, 0)", "Monday")] + [InlineData("WEEKDAYNAME(2, TRUE, 3)", "Wed")] + [InlineData("DATEVALUE('12:30')", "1899-12-30 00:00:00")] + [InlineData("TIMEVALUE('2020-01-02')", "1899-12-30 00:00:00")] + [InlineData("ISDATE('1,000')", "True")] + [InlineData("ISDATE('Feb 30')", "True")] + [InlineData("ISDATE('2020-01-02T12:00')", "False")] + public void Names_and_date_text(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression))); + + [Theory] + [InlineData("WEEKDAYNAME(1)", "Sunday")] + [InlineData("WEEKDAY(#2020-01-02#, 0)", "5")] + public void The_system_first_day_follows_the_culture(string expression, string expected) => + Assert.Equal(expected, Text(Scalar(expression, CultureInfo.GetCultureInfo("en-US")))); + + [Theory] + [InlineData("WEEKDAY(DT, 8)")] + [InlineData("WEEKDAY(DT, -1)")] + [InlineData("DATEPART('ww', DT, 8)")] + [InlineData("DATEPART('ww', DT, 1, 4)")] + [InlineData("DATEPART('x', DT)")] + [InlineData("DATEADD('x', 1, DT)")] + [InlineData("DATEADD('yyyy', -1921, DT)")] + [InlineData("DATEDIFF('ww', #2020-01-01#, #2020-02-01#, 8)")] + [InlineData("DATESERIAL(10000, 1, 1)")] + [InlineData("DATESERIAL(-8100, 1, 1)")] + [InlineData("MONTHNAME(13)")] + [InlineData("MONTHNAME(TRUE)")] + [InlineData("WEEKDAYNAME(0)")] + [InlineData("WEEKDAYNAME(8)")] + [InlineData("WEEKDAYNAME(TRUE)")] + [InlineData("WEEKDAYNAME(1, FALSE, 8)")] + public void Out_of_range_arguments_are_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("DATESERIAL(2020, 1, 32768)")] + [InlineData("TIMESERIAL(0, 0, 32768)")] + [InlineData("DATEDIFF('s', #0100-01-01#, #9999-12-31#)")] + [InlineData("DATEDIFF('n', #0100-01-01#, #9999-12-31#)")] + public void Values_past_their_type_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("DATEVALUE(43832.5)")] + [InlineData("TIMEVALUE(43832.5)")] + [InlineData("DATEVALUE('43832.5')")] + [InlineData("DATEVALUE(TRUE)")] + [InlineData("DATEVALUE(CCUR(43832.25))")] + [InlineData("TIMEVALUE('$5')")] + [InlineData("MONTHNAME(1, 'x')")] + public void Datevalue_and_timevalue_take_only_dates_and_date_text(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("WEEKDAY(DT, NULL)")] + [InlineData("DATEPART('ww', DT, NULL)")] + [InlineData("DATEADD('m', NULL, DT)")] + [InlineData("DATEADD('m', 1, NULL)")] + [InlineData("DATEDIFF('yyyy', NULL, DT)")] + [InlineData("DATESERIAL(NULL, 1, 1)")] + [InlineData("TIMESERIAL(NULL, 1, 1)")] + [InlineData("MONTHNAME(NULL)")] + [InlineData("MONTHNAME(1, NULL)")] + [InlineData("WEEKDAYNAME(1, NULL)")] + [InlineData("WEEKDAYNAME(1, FALSE, NULL)")] + [InlineData("DATEVALUE(NULL)")] + [InlineData("TIMEVALUE(NT)")] + public void A_null_argument_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + // ACE rounds a date to the second; LibRed keeps milliseconds, so 0.864 seconds is still second 0. + [Fact] + public void Second_keeps_the_fraction_of_a_second() => + Assert.Equal(0, Scalar("SECOND(0.00001)")); +} diff --git a/test/LibRed.Engine.Tests/DdlDmlTests.cs b/test/LibRed.Engine.Tests/DdlDmlTests.cs index faa8dc319..ab3b91b73 100644 --- a/test/LibRed.Engine.Tests/DdlDmlTests.cs +++ b/test/LibRed.Engine.Tests/DdlDmlTests.cs @@ -199,6 +199,32 @@ public void Datetime2_round_trips_through_libred_on_the_ace17_format() finally { TemporaryDatabase.Delete(path); } } + // A DATETIME2 keeps 100-ns ticks, and three values inside one millisecond stay three values: they sort by tick, + // and equality and ranges see the ticks too — comparing by the millisecond OA serial alone made them all equal. + [Fact] + public void Datetime2_values_within_one_millisecond_stay_distinct() + { + var millisecond = new DateTime(2021, 3, 4, 5, 6, 7, 8); + string path = CopyToTemp(); + try + { + SetVersionByte(path, 0x06); + using var db = JetDatabase.Open(path, readOnly: false); + var e = new QueryEngine(db); + e.ExecuteNonQuery("CREATE TABLE `E` (`Id` INTEGER PRIMARY KEY, `V` DATETIME2 NULL)"); + foreach ((int id, int ticks) in new[] { (1, 3), (2, 1), (3, 2) }) + e.ExecuteNonQuery("INSERT INTO `E` (`Id`, `V`) VALUES (@id, @v)", + new Dictionary { ["id"] = id, ["v"] = millisecond.AddTicks(ticks) }); + + var middle = new Dictionary { ["v"] = millisecond.AddTicks(2) }; + Assert.Equal([2, 3, 1], e.ExecuteQuery("SELECT `Id` FROM `E` ORDER BY `V`").Rows.Select(r => Convert.ToInt32(r[0]))); + Assert.Equal([3], e.ExecuteQuery("SELECT `Id` FROM `E` WHERE `V` = @v", middle).Rows.Select(r => Convert.ToInt32(r[0]))); + Assert.Equal([1], e.ExecuteQuery("SELECT `Id` FROM `E` WHERE `V` > @v", middle).Rows.Select(r => Convert.ToInt32(r[0]))); + Assert.Equal(3, e.ExecuteQuery("SELECT COUNT(*) FROM (SELECT DISTINCT `V` FROM `E`) AS `D`").Rows.Single()[0]); + } + finally { TemporaryDatabase.Delete(path); } + } + /// Raises a copied file to the ACE 17 format. Page 0 offset 0x14 is the entire upgrade — see /// docs/format/page-00-database.md and AceDateTime2UpgradeTests. private static void SetVersionByte(string path, byte version) diff --git a/test/LibRed.Engine.Tests/DecimalWriteTests.cs b/test/LibRed.Engine.Tests/DecimalWriteTests.cs new file mode 100644 index 000000000..4c056e8f1 --- /dev/null +++ b/test/LibRed.Engine.Tests/DecimalWriteTests.cs @@ -0,0 +1,57 @@ +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// A value written to a DECIMAL column keeps its fraction, from a parameter or a literal. +public class DecimalWriteTests : TempDatabaseTest +{ + private static QueryEngine Fresh() + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "decimal-write-"); + var engine = new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); + engine.ExecuteNonQuery("CREATE TABLE Z (Id LONG, V DECIMAL(18,2))"); + return engine; + } + + [Theory] + [InlineData("-1234567890.01")] + [InlineData("4.5")] + [InlineData("0.01")] + public void A_parameter_keeps_its_fraction(string text) + { + QueryEngine engine = Fresh(); + decimal value = decimal.Parse(text, System.Globalization.CultureInfo.InvariantCulture); + engine.ExecuteNonQuery("INSERT INTO Z (Id, V) VALUES (1, @p)", new Dictionary { ["@p"] = value }); + Assert.Equal(value, engine.ExecuteQuery("SELECT V FROM Z").Rows.Single()[0]); + Assert.Equal(value.ToString(System.Globalization.CultureInfo.CurrentCulture).TrimEnd('0').TrimEnd('.'), + engine.ExecuteQuery("SELECT V & '' FROM Z").Rows.Single()[0]); + } + + [Fact] + public void A_literal_keeps_its_fraction() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("INSERT INTO Z (Id, V) VALUES (1, -1234567890.01)"); + Assert.Equal(-1234567890.01m, engine.ExecuteQuery("SELECT V FROM Z").Rows.Single()[0]); + } + + [Fact] + public void An_update_keeps_its_fraction() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("INSERT INTO Z (Id, V) VALUES (1, 0)"); + engine.ExecuteNonQuery("UPDATE Z SET V = -1234567890.01"); + Assert.Equal(-1234567890.01m, engine.ExecuteQuery("SELECT V FROM Z").Rows.Single()[0]); + } + + [Fact] + public void A_currency_literal_keeps_its_fraction() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("CREATE TABLE M (Id LONG, V CURRENCY)"); + engine.ExecuteNonQuery("INSERT INTO M (Id, V) VALUES (1, -92233720368.4775)"); + Assert.Equal(-92233720368.4775m, engine.ExecuteQuery("SELECT V FROM M").Rows.Single()[0]); + } +} diff --git a/test/LibRed.Engine.Tests/DeferredFunctionsTests.cs b/test/LibRed.Engine.Tests/DeferredFunctionsTests.cs index d4297808c..b81e14375 100644 --- a/test/LibRed.Engine.Tests/DeferredFunctionsTests.cs +++ b/test/LibRed.Engine.Tests/DeferredFunctionsTests.cs @@ -42,7 +42,7 @@ public void StrConv_case_modes_match_ace(string expr, string expected) [Fact] public void StrConv_unsupported_mode_is_rejected() { - var ex = Assert.Throws(() => Eval("StrConv('hello', 4)")); + var ex = Assert.Throws(() => Eval("StrConv('hello', 4)")); Assert.Contains("Invalid procedure call", ex.Message); } @@ -57,9 +57,4 @@ public void StrConv_unsupported_mode_is_rejected() [InlineData("WeekdayName(3, True, 2)", "Wed")] public void WeekdayName_with_explicit_first_day_matches_ace(string expr, string expected) => Assert.Equal(expected, Eval(expr)); - - [Fact] - public void WeekdayName_omitted_first_day_defaults_to_sunday() - // ACE's omitted default follows the OS regional first day; LibRed fixes it to vbSunday for determinism. - => Assert.Equal("Sunday", Eval("WeekdayName(1)")); } diff --git a/test/LibRed.Engine.Tests/FinancialFunctionsTests.cs b/test/LibRed.Engine.Tests/FinancialFunctionsTests.cs index 8a99c43b5..c00ad340f 100644 --- a/test/LibRed.Engine.Tests/FinancialFunctionsTests.cs +++ b/test/LibRed.Engine.Tests/FinancialFunctionsTests.cs @@ -7,25 +7,20 @@ namespace LibRed.Engine.Tests; // Financial, FormatX, and colour functions — all exposed by the ACE JES and now implemented in LibRed. Expected // values are exactly what ACE returned. Culture pinned to en-US for the locale-sensitive FormatX cases. -public class FinancialFunctionsTests : TempDatabaseTest +public class FinancialFunctionsTests(FinancialFunctionsTests.Database database) + : TempDatabaseTest, IClassFixture { - private static QueryEngine Fresh() - { - string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "fin-"); - var e = new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); - e.ExecuteNonQuery("CREATE TABLE T ( K LONG PRIMARY KEY )"); - e.ExecuteNonQuery("INSERT INTO T (K) VALUES (1)"); - return e; - } - - private static object? Eval(string expr) => Fresh().ExecuteQuery($"SELECT {expr} FROM T").Rows.Single()[0]; - - private static object? EvalEnUs(string expr) - { - var prev = CultureInfo.CurrentCulture; - CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("en-US"); - try { return Eval(expr); } finally { CultureInfo.CurrentCulture = prev; } - } + private static readonly string[] Setup = + [ + "CREATE TABLE T ( K LONG PRIMARY KEY )", + "INSERT INTO T (K) VALUES (1)", + ]; + + public sealed class Database() : SharedDatabase("fin-", Setup); + + private object? Eval(string expr) => database.Engine.ExecuteQuery($"SELECT {expr} FROM T").Rows.Single()[0]; + + private object? EvalEnUs(string expr) => database.Scalar($"SELECT {expr} FROM T", CultureInfo.GetCultureInfo("en-US")); [Theory] [InlineData("Pmt(0.05/12, 60, 10000)", -188.7123364401099)] @@ -41,6 +36,97 @@ private static QueryEngine Fresh() public void Financial_matches_ace(string expr, double expected) => Assert.Equal(expected, Convert.ToDouble(Eval(expr)), 6); + // Exact to the last bit, as the VBA runtime's order of arithmetic gives them. Rate(60, -100, 10000) is the one + // probed case that still differs, in its last digits (-0.01544514669212213 vs ACE's -0.015445146692122057), + // most likely from the platform's Pow. + [Theory] + [InlineData("NPER(0.05/12, -200, 10000)", 56.18429076143198)] + [InlineData("FV(-0.05, 60, -200)", 3815.720804052192)] + [InlineData("PV(-0.05, 60, -200)", 82824.77649040522)] + [InlineData("FV(0.01, 60, -200, -1000, 1)", 18313.970009558736)] + [InlineData("PV(0.01, 60, -200, 1000, 1)", 8530.468142119496)] + [InlineData("PMT(0.01, 60, 10000, 500, 1)", -226.3036640509589)] + [InlineData("PMT(0.01, 60, 10000, 0, 2)", -220.24205628615604)] + [InlineData("PMT(0.01, 60, 10000, 0, -1)", -220.24205628615604)] + [InlineData("PMT(0.01, 60, 10000, 0, '1')", -220.24205628615604)] + [InlineData("PMT(0.01, 60, 10000, 0, 0.5)", -222.44447684901763)] + [InlineData("PMT(0.01, -60, 10000)", 122.44447684901762)] + [InlineData("PMT(TRUE, 60, 10000)", -0.0)] + [InlineData("PMT('0.01', 60, 10000)", -222.44447684901763)] + [InlineData("NPER(0.01, -200, 10000, 0, 1)", 68.67056927050619)] + [InlineData("NPER(0.01, 200, 10000)", -40.748907156094134)] + [InlineData("NPER(0.01, -200, 0)", 0.0)] + [InlineData("IPMT(0.01, 2, 60, 10000, 0, 1)", -97.79757943713845)] + [InlineData("IPMT(0.01, 1, 60, 10000, 0, 1)", 0.0)] + [InlineData("IPMT(0.01, 60.5, 60, 10000)", -1.1039496274401608)] + [InlineData("PPMT(0.01, 1, 60, 10000, 0, 1)", -220.24205628615604)] + [InlineData("PPMT(0.01, 60, 60, 10000, 500, 1)", -228.96451396085524)] + [InlineData("RATE(60, -200, 10000)", 0.006183413161266263)] + [InlineData("RATE(60, -200, 10000, 0, 1)", 0.006407985777795751)] + [InlineData("RATE(60, -200, 10000, 0, 0, 0)", 0.0061834131612155535)] + [InlineData("RATE(1, -200, 100)", 0.9999999999999825)] + [InlineData("RATE(12, -1000, 10000, 0, 0, 0.9)", 0.029228540769573003)] + [InlineData("SLN(10000, 1000, TRUE)", -9000.0)] + [InlineData("SYD(10000, 1000, 5.5, 1)", 2769.230769230769)] + [InlineData("SYD(-10000, 1000, 5, 1)", -3666.6666666666665)] + [InlineData("SYD(10000, 1000, 5, 2.5)", 2100.0)] + [InlineData("DDB(10000, 1000, 5, 5)", 295.9999999999985)] + [InlineData("DDB(10000, 1000, 5, 1.5)", 3098.3866769659335)] + [InlineData("DDB(10000, 1000, 5, 0.5)", 4000.0)] + [InlineData("DDB(10000, 1000, 3, 1)", 6666.666666666667)] + [InlineData("DDB(10000, 20000, 5, 1)", -10000.0)] + [InlineData("DDB(10000, 1000, 5, 1, 10)", 9000.0)] + [InlineData("DDB(10000, 1000, 5, 2, 5)", 0.0)] + [InlineData("DDB(-10000, 1000, 5, 1)", 0.0)] + [InlineData("PV(-1, 60, -200)", double.PositiveInfinity)] + [InlineData("PMT(10, 1000, 10000)", double.NaN)] + public void Financial_matches_ace_exactly(string expr, double expected) + => Assert.Equal(expected, Assert.IsType(Eval(expr))); + + [Theory] + [InlineData("PMT(0.01, 0, 10000)")] + [InlineData("NPER(-1, -200, 10000)")] + [InlineData("NPER(0, 0, 10000)")] + [InlineData("NPER(0.01, 0, 10000)")] + [InlineData("NPER(0.01, -50, 10000)")] + [InlineData("NPER(0.5, -200, 10000)")] + [InlineData("IPMT(0.01, 0, 60, 10000)")] + [InlineData("IPMT(0.01, 61, 60, 10000)")] + [InlineData("IPMT(0.01, 2, 0, 10000)")] + [InlineData("PPMT(0.01, 61, 60, 10000)")] + [InlineData("RATE(0, -200, 10000)")] + [InlineData("RATE(60, 200, 10000)")] + [InlineData("RATE(60, 0, 10000)")] + [InlineData("RATE(60, -200, 10000, 0, 0, 10)")] + [InlineData("SLN(10000, 1000, 0)")] + [InlineData("SYD(10000, 1000, 5, 6)")] + [InlineData("SYD(10000, 1000, 5, 0)")] + [InlineData("SYD(10000, -1000, 5, 1)")] + [InlineData("DDB(10000, 1000, 5, 6)")] + [InlineData("DDB(10000, 1000, 5, 5.5)")] + [InlineData("DDB(10000, 1000, 5, 0)")] + [InlineData("DDB(10000, 1000, 5, 1, 0)")] + [InlineData("DDB(10000, 1000, 0, 1)")] + [InlineData("DDB(10000, -1000, 5, 1)")] + public void Out_of_range_arguments_are_an_invalid_procedure_call(string expr) + => Assert.Throws(() => Eval(expr)); + + [Theory] + [InlineData("PMT(0.01, 60, 'abc')")] + public void Text_that_is_not_a_number_is_a_type_mismatch(string expr) + => Assert.Throws(() => Eval(expr)); + + // ACE raises "Data type mismatch" or "Invalid use of Null"; LibRed returns Null. + [Theory] + [InlineData("PMT(NULL, 60, 10000)")] + [InlineData("PMT(0.01, 60, 10000, NULL)")] + [InlineData("IPMT(0.01, NULL, 60, 10000)")] + [InlineData("RATE(60, -200, 10000, 0, 0, NULL)")] + [InlineData("SLN(NULL, 1000, 5)")] + [InlineData("DDB(10000, 1000, 5, 1, NULL)")] + public void A_null_argument_gives_null(string expr) + => Assert.Null(Eval(expr)); + [Theory] [InlineData("RGB(255, 0, 0)", 255)] [InlineData("RGB(0, 255, 0)", 65280)] diff --git a/test/LibRed.Engine.Tests/FormatFunctionTests.cs b/test/LibRed.Engine.Tests/FormatFunctionTests.cs index 89fc0e537..6670bcaa2 100644 --- a/test/LibRed.Engine.Tests/FormatFunctionTests.cs +++ b/test/LibRed.Engine.Tests/FormatFunctionTests.cs @@ -1,65 +1,307 @@ using System.Globalization; -using LibRed; using LibRed.Engine; using Xunit; namespace LibRed.Engine.Tests; -// Access Format(value, format). Named formats and custom numeric/date/string strings, driven off the current -// culture (as ACE drives them off the OS regional settings). Culture is pinned to en-US here so the -// locale-dependent named date/currency formats are deterministic; the values match what ACE produces on an -// en-US host (and the culture-invariant custom strings match ACE on any host). -public class FormatFunctionTests +/// +/// Format FormatNumber FormatCurrency FormatPercent FormatDateTime. The expected +/// values were measured against ACE (under en-AU, and written here for en-US, which differs only in its date order and +/// long date), except that a Null format or setting gives Null where ACE raises an error. +/// +public class FormatFunctionTests(FormatFunctionTests.Database database) + : TempDatabaseTest, IClassFixture { - private static string EvalEnUs(string expr) - { - var prev = CultureInfo.CurrentCulture; - CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("en-US"); - try - { - using var temp = TemporaryDatabase.CopyOf( - Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "fmt-"); - var e = new QueryEngine(temp.Open()); - e.ExecuteNonQuery("CREATE TABLE T ( K LONG PRIMARY KEY )"); - e.ExecuteNonQuery("INSERT INTO T (K) VALUES (1)"); - string r = e.ExecuteQuery($"SELECT {expr} FROM T").Rows.Single()[0]?.ToString()!; - return r; - } - finally { CultureInfo.CurrentCulture = prev; } - } + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, DT DATETIME)", + "INSERT INTO T (Id, DT) VALUES (1, #2020-01-02 12:00:00#)", + ]; + + public sealed class Database() : SharedDatabase("format-", Setup); + + // Formats follow the regional settings, so each query runs under en-US whatever the machine's culture. + private object? Scalar(string expression) => + database.Scalar($"SELECT {expression} FROM T", CultureInfo.GetCultureInfo("en-US")); + + private void AssertFormat(string value, string format, string expected) => + Assert.Equal(expected, Scalar($"FORMAT({value}, \"{format.Replace("\"", "\"\"")}\")")); + + [Theory] + [InlineData("-1234.5678", "#,##0.00", "-1,234.57")] + [InlineData("0", "#.##", ".")] + [InlineData("0.5", "#.##", ".5")] + [InlineData("-0.005", "0.00", "-0.01")] + [InlineData("-0.005", "0", "0")] + [InlineData("CDBL(0.125)", "0.00", "0.13")] + [InlineData("CDBL(1.005)", "0.00", "1.01")] + [InlineData("CDBL(2.5)", "0", "3")] + [InlineData("CDBL(1) / 3", "0.00000000000000000000", "0.33333333333333300000")] + [InlineData("CDBL(1234567890123456789)", "0", "1234567890123460000")] + [InlineData("CSNG(0.1)", "0.0000000000", "0.1000000000")] + [InlineData("1234.5678", "0.0#", "1234.57")] + [InlineData("1234.5", "0.0#", "1234.5")] + [InlineData("12.3", "0.0#0", "12.300")] + [InlineData("12.3", "#0.#0#", "12.30")] + [InlineData("12", "0.#", "12.")] + [InlineData("1234.5", "0.0.0", "1234.5.0")] + [InlineData("1234.5", "0.0 0", "1234.5 0")] + [InlineData("1234.5678", "0.5", "1235.5")] + [InlineData("TRUE", "0.00", "-1.00")] + [InlineData("DT", "0.00", "43832.50")] + [InlineData("'12'", "0.00", "12.00")] + [InlineData("'2/1/2020'", "0.00", "43862.00")] + [InlineData("'abc'", "0.00", "abc")] + [InlineData("''", "0", "")] + public void Digit_placeholders_round_half_away_from_zero(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("123456789", "#,##0,", "123,457")] + [InlineData("123456789", "#,##0,,", "123")] + [InlineData("-1234.5678", "#,##0,,", "0")] + [InlineData("1234567", "0,.0", "1234.6")] + [InlineData("1234567", "#,##0, \"k\"", "1,235 k")] + [InlineData("1234.5678", "0.00,", "1234.57")] + [InlineData("1234", ",0", ",1234")] + [InlineData("1234", "0,0", "1,234")] + [InlineData("12345", "#,#0:0", "12,34:5")] + [InlineData("123456789", "00:00", "1234567:89")] + [InlineData("-1", "00:00", "-00:01")] + [InlineData("5", "00 00", "00 05")] + [InlineData("0", "#.#%", ".%")] + [InlineData("1234.5678", "0.0%%", "12345678.0%%")] + [InlineData("1234.5678", "0.00\\%", "1234.57%")] + public void Commas_group_or_scale_and_literals_sit_among_the_digits(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("-1234.5678", "0.00E-00", "-1.23E03")] + [InlineData("-1234.5678", "0.00e+00", "-1.23e+03")] + [InlineData("1234.5678", "##0.0E+0", "123.5E+1")] + [InlineData("0", "##0.0E+0", "000.0E+0")] + [InlineData("1234.5", "#,##0.00E+00", "1,234.50E+00")] + [InlineData("1234.5", "0.00E+", "1.23E+3")] + [InlineData("1234.5", "#.##E+##", "1.23E+3")] + [InlineData("0", "#.##E+##", "0.E+0")] + [InlineData("999.99", "0.0E+0", "1.0E+3")] + [InlineData("0.00012", "0.0E-0", "1.2E-4")] + [InlineData("1234.5", "0.0E+0%", "1.2E+5%")] + [InlineData("1234.5", "E+0", "E+1235")] + [InlineData("1234.5", "0E", "1235")] + public void Exponent_formats_keep_the_integer_placeholders(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("-1234.5678", "$#,##0.00;($#,##0.00)", "($1,234.57)")] + [InlineData("0", "0;-0;\"zero\"", "zero")] + [InlineData("-0.005", "0;-0;\"zero\"", "zero")] + [InlineData("-1", "0;;\"z\";\"null\"", "-1")] + [InlineData("NULL", "0;;\"z\";\"null\"", "null")] + [InlineData("NULL", "0", "")] + [InlineData("-1234.5678", ";;;", "")] + [InlineData("-1", "0;", "-1")] + [InlineData("-0.001", "0.00;(0.00)", "0.00")] + [InlineData("0.001", "0.00;(0.00);\"z\"", "z")] + [InlineData("-0.001", "#.##;;\"z\"", "z")] + [InlineData("-0.001", "x;;\"z\"", "-x")] + [InlineData("-0.4", "0;x;\"z\"", "x")] + [InlineData("0.4", "0;;", "0")] + [InlineData("1", ";x", "")] + [InlineData("-1", ";x", "x")] + [InlineData("-1", "x", "-x")] + [InlineData("-1", "-0", "--1")] + [InlineData("-5", "(0)", "-(5)")] + [InlineData("-1", "\\$0", "-$1")] + [InlineData("-1", "\"x\"0", "-x1")] + [InlineData("5", "[Red]0", "5")] + [InlineData("5", "*0", "")] + public void Sections_choose_by_sign_and_rounded_zero(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("-1234.5678", "General Number", "-1234.5678")] + [InlineData("DT", "General Number", "43832.5")] + [InlineData("CDBL(1) / 3", "General Number", "0.333333333333333")] + [InlineData("-1234.5678", "Currency", "-$1,234.57")] + [InlineData("-0.001", "Currency", "$0.00")] + [InlineData("0.5", "Fixed", "0.50")] + [InlineData("1234.5", "fixed", "1234.50")] + [InlineData("-0.001", "Fixed", "0.00")] + [InlineData("1234567.891", "Standard", "1,234,567.89")] + [InlineData("-0.001", "Percent", "-0.10%")] + [InlineData("-0.001", "Scientific", "-1.00E-03")] + [InlineData("TRUE", "Yes/No", "Yes")] + [InlineData("#1899-12-30#", "Yes/No", "No")] + [InlineData("'0'", "On/Off", "Off")] + [InlineData("'abc'", "True/False", "abc")] + [InlineData("1234.5", " Fixed", " Fixe18")] + [InlineData("DT", "General Date", "1/2/2020 12:00:00 PM")] + [InlineData("0", "General Date", "12:00:00 AM")] + [InlineData("TRUE", "General Date", "12/29/1899")] + [InlineData("DT", "Long Date", "Thursday, January 2, 2020")] + [InlineData("DT", "Medium Date", "02-Jan-20")] + [InlineData("#0100-01-01#", "Short Date", "1/1/0100")] + [InlineData("#13:45:30#", "Long Time", "1:45:30 PM")] + [InlineData("#13:45:30#", "Medium Time", "01:45 PM")] + [InlineData("#13:45:30#", "Short Time", "13:45")] + [InlineData("DT", "", "1/2/2020 12:00:00 PM")] + [InlineData("#1899-12-30#", "", "12:00:00 AM")] + [InlineData("#0100-01-01#", "", "1/1/0100")] + [InlineData("TRUE", "", "-1")] + public void Named_formats_match_whole_and_ignore_case(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("DT", "yyyy-mm-dd", "2020-01-02")] + [InlineData("DT", "dddd, mmmm d, yyyy", "Thursday, January 2, 2020")] + [InlineData("DT", "m", "1")] + [InlineData("DT", "hh:mm:ss", "12:00:00")] + [InlineData("DT", "mmm hh", "Jan 12")] + [InlineData("DT", "h mmm", "12 Jan")] + [InlineData("DT", "hh \"x\" mm", "12 x 00")] + [InlineData("DT", "n m", "0 1")] + [InlineData("DT", "hh:mm:ss mm", "12:00:00 01")] + [InlineData("DT", "q/y/w/ww", "1/2/5/1")] + [InlineData("DT", "HH:NN", "12:00")] + [InlineData("DT", "tt", "tt")] + [InlineData("DT", "tttttt", "12:00:00 PMt")] + [InlineData("DT", "d\\d", "2d")] + [InlineData("DT", "\"hh\"", "hh")] + [InlineData("#13:45:30#", "h:n:s", "13:45:30")] + [InlineData("#13:45:30#", "hh:mm AM/PM", "01:45 PM")] + [InlineData("#13:45:30#", "hh am/pm", "01 pm")] + [InlineData("#13:45:30#", "h:nn a/p", "1:45 p")] + [InlineData("#13:45:30#", "h:nn A/P", "1:45 P")] + [InlineData("#13:45:30#", "h AMPM", "1 PM")] + [InlineData("#13:45:30#", "c", "1:45:30 PM")] + [InlineData("-1234.5678", "yyy", "96226")] + [InlineData("-1234.5678", "mmmmm", "August8")] + [InlineData("-1234.5678", "hhh", "1313")] + [InlineData("-1234.5678", "sss", "3838")] + [InlineData("-1234.5678", "d.m.y", "13.8.226")] + [InlineData("-1234.5678", "hh:nn:ss", "13:37:38")] + [InlineData("CDATE(43832.9999999)", "yyyy-mm-dd hh:nn:ss", "2020-01-03 00:00:00")] + [InlineData("#0100-01-01#", "yyyy", "0100")] + [InlineData("0", "/", "/")] + [InlineData("-1", ":", ":")] + [InlineData("'2/1/2020'", "yyyy-mm-dd", "2020-02-01")] + [InlineData("'abc'", "yyyy", "abc")] + [InlineData("''", "yyyy", "")] + [InlineData("NULL", "yyyy", "")] + [InlineData("DT", "dd/mm/yyyy;x", "02/01/2020")] + public void Date_symbols_are_read_longest_first(string value, string format, string expected) => + AssertFormat(value, format, expected); [Theory] - // custom numeric - [InlineData("Format(1234.5, '0.00')", "1234.50")] - [InlineData("Format(1234.5, '#,##0.00')", "1,234.50")] - [InlineData("Format(0.25, '0%')", "25%")] - [InlineData("Format(5, '000')", "005")] - [InlineData("Format(-5, '000')", "-005")] - [InlineData("Format(255, '\\#0')", "#255")] - // named numeric / boolean - [InlineData("Format(1234.5, 'Currency')", "$1,234.50")] - [InlineData("Format(1234.5, 'Fixed')", "1234.50")] - [InlineData("Format(1234.5, 'Standard')", "1,234.50")] - [InlineData("Format(0.25, 'Percent')", "25.00%")] - [InlineData("Format(1234.5, 'Scientific')", "1.23E+03")] - [InlineData("Format(1234.5, 'General Number')", "1234.5")] - [InlineData("Format(1, 'Yes/No')", "Yes")] - [InlineData("Format(0, 'Yes/No')", "No")] - [InlineData("Format(0, 'True/False')", "False")] - [InlineData("Format(1, 'On/Off')", "On")] - // custom date (culture-invariant tokens) - [InlineData("Format(#2020-06-15 13:05:09#, 'yyyy-mm-dd')", "2020-06-15")] - [InlineData("Format(#2020-06-15 13:05:09#, 'hh:nn:ss')", "13:05:09")] - [InlineData("Format(#2020-06-15 13:05:09#, 'mmmm d, yyyy')", "June 15, 2020")] - [InlineData("Format(#2020-06-15 13:05:09#, 'ddd')", "Mon")] - [InlineData("Format(#2020-06-15#, 'q')", "2")] - // named date/time (en-US) - [InlineData("Format(#2020-06-15 13:05:09#, 'Short Date')", "6/15/2020")] - [InlineData("Format(#2020-06-15 13:05:09#, 'Long Date')", "Monday, June 15, 2020")] - [InlineData("Format(#2020-06-15 13:05:09#, 'Medium Time')", "01:05 PM")] - // string - [InlineData("Format('hello', '>')", "HELLO")] - [InlineData("Format('hello', '<')", "hello")] - public void Format_matches_ace_under_en_us(string expr, string expected) - => Assert.Equal(expected, EvalEnUs(expr)); + [InlineData("'abc'", "@@@@@@", " abc")] + [InlineData("'abc'", "!@@@@@@", "abc ")] + [InlineData("-1234.5678", "!@@@@@@", "4.5678")] + [InlineData("'abc'", ">", "ABC")] + [InlineData("'abc'", "<>", "abc")] + [InlineData("'abc'", "@\"x\"", "axbc")] + [InlineData("'ab'", "@&@&", " ab")] + [InlineData("'ab'", "!@&@&", "ab ")] + [InlineData("'abc'", ">\"x\"", "xABC")] + [InlineData("'abc'", "\\@@", "@abc")] + [InlineData("''", "@;\"empty\"", "empty")] + [InlineData("NULL", "@;\"empty\"", "empty")] + [InlineData("NULL", "@;@", " ")] + [InlineData("0", "(@@@) @@@-@@@@", "( ) - 0")] + [InlineData("-1234.5678", "(@@@) @@@-@@@@", "(-12) 34.-5678")] + [InlineData("-1234.5678", "@@-@@", "-1-234.5678")] + [InlineData("0", "<@@@@@@@", " 0")] + [InlineData("TRUE", "@@@", " -1")] + [InlineData("DT", "<", "1/2/2020 12:00:00 pm")] + public void Text_placeholders_fill_from_the_right(string value, string format, string expected) => + AssertFormat(value, format, expected); + + [Theory] + [InlineData("FORMAT(DT, 'w', 2)", "4")] + [InlineData("FORMAT(DT, 'ww', 2)", "1")] + [InlineData("FORMAT(#2021-01-01#, 'ww', 1, 2)", "53")] + [InlineData("FORMAT(#2021-01-01#, 'ww', 2, 3)", "52")] + [InlineData("FORMAT(1234.5)", "1234.5")] + [InlineData("FORMAT(NULL)", "")] + [InlineData("FORMAT('abc', '@@@@@', 2)", " abc")] + public void Format_takes_week_settings(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("FORMATNUMBER(1234.5)", "1,234.50")] + [InlineData("FORMATNUMBER(0.125)", "0.13")] + [InlineData("FORMATNUMBER(2.5, 0)", "3")] + [InlineData("FORMATNUMBER(-0.5, 0)", "-1")] + [InlineData("FORMATNUMBER(-0.001, 2)", "0.00")] + [InlineData("FORMATNUMBER(0.5, 2, 0)", ".50")] + [InlineData("FORMATNUMBER(-1234.5678, 2, -2, -1)", "(1,234.57)")] + [InlineData("FORMATNUMBER(-1234.5678, 2, -2, -2, 0)", "-1234.57")] + [InlineData("FORMATNUMBER(1, 2.5)", "1.00")] + [InlineData("FORMATNUMBER(TRUE)", "-1.00")] + [InlineData("FORMATNUMBER(DT)", "43,832.50")] + [InlineData("FORMATNUMBER(NULL)", "")] + [InlineData("FORMATCURRENCY(1234.5)", "$1,234.50")] + [InlineData("FORMATCURRENCY(0, 0)", "$0")] + [InlineData("FORMATCURRENCY(0.5, 2, 0)", "$.50")] + [InlineData("FORMATCURRENCY(-1234.5678, 2, -2, -1)", "($1,234.57)")] + [InlineData("FORMATCURRENCY(-1234.5678, 2, -2, -2, 0)", "-$1234.57")] + [InlineData("FORMATCURRENCY(-0.001, 2)", "$0.00")] + [InlineData("FORMATPERCENT(0.25)", "25.00%")] + [InlineData("FORMATPERCENT(1234.5678)", "123,456.78%")] + [InlineData("FORMATPERCENT(-0.5, 2, -2, -1)", "(50.00%)")] + [InlineData("FORMATPERCENT('12.5')", "1,250.00%")] + [InlineData("FORMATPERCENT(-0.00001)", "0.00%")] + public void Formatnumber_currency_and_percent_round_half_away_from_zero(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("FORMATDATETIME(DT)", "1/2/2020 12:00:00 PM")] + [InlineData("FORMATDATETIME(#2020-06-15#)", "6/15/2020")] + [InlineData("FORMATDATETIME(#13:45:30#)", "1:45:30 PM")] + [InlineData("FORMATDATETIME(#13:45:30#, 1)", "Saturday, December 30, 1899")] + [InlineData("FORMATDATETIME(DT, 2)", "1/2/2020")] + [InlineData("FORMATDATETIME(DT, 3)", "12:00:00 PM")] + [InlineData("FORMATDATETIME(#13:45:30#, 4)", "13:45")] + [InlineData("FORMATDATETIME(#13:45:30#, 1.5)", "12/30/1899")] + [InlineData("FORMATDATETIME(43832.5, '2')", "1/2/2020")] + [InlineData("FORMATDATETIME(#0100-01-01#, 2)", "1/1/0100")] + [InlineData("FORMATDATETIME(NULL, 3)", "")] + public void Formatdatetime_writes_the_named_formats(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("FORMAT(1, '0', 8)")] + [InlineData("FORMAT(1, 'Currency', 2, 4)")] + [InlineData("FORMATNUMBER(1, -2)")] + [InlineData("FORMATNUMBER(1, 2, 1)")] + [InlineData("FORMATCURRENCY(1, 2, 5)")] + [InlineData("FORMATDATETIME(DT, 5)")] + [InlineData("FORMATDATETIME(NULL, -1)")] + public void Out_of_range_settings_are_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("FORMAT(1E+300, 'yyyy')")] + [InlineData("FORMAT(123456789, '/')")] + public void A_number_past_a_date_overflows_a_date_format(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("FORMATNUMBER('abc')")] + [InlineData("FORMATDATETIME('abc')")] + public void Text_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("FORMAT(DT, NULL)")] + [InlineData("FORMAT(DT, 'w', NULL)")] + [InlineData("FORMATNUMBER(1, NULL)")] + [InlineData("FORMATCURRENCY(1, 2, NULL)")] + [InlineData("FORMATDATETIME(DT, NULL)")] + public void A_null_setting_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + [Fact] + public void A_time_literal_is_on_day_zero() => + Assert.Equal(new DateTime(1899, 12, 30, 13, 45, 30), Scalar("#13:45:30#")); } diff --git a/test/LibRed.Engine.Tests/GreatestLeastTests.cs b/test/LibRed.Engine.Tests/GreatestLeastTests.cs new file mode 100644 index 000000000..a921f7de9 --- /dev/null +++ b/test/LibRed.Engine.Tests/GreatestLeastTests.cs @@ -0,0 +1,125 @@ +using System.Linq; +using LibRed; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// GREATEST(a, b, …) and LEAST(a, b, …) — the largest and smallest argument, NULLs ignored. +/// Access/ACE has neither, so like COALESCE they are reachable from LibRed's extended SQL mode, which translates +/// Math.Max/Math.Min to them, and from hand-written SQL. +/// +public class GreatestLeastTests +{ + private static QueryEngine Northwind() + { + string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "greatest-"); + return new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); + } + + private static object? Scalar(string expr) + => Northwind().ExecuteQuery($"SELECT {expr} FROM Employees WHERE EmployeeID = 1").Rows.First()[0]; + + private static Type ColumnType(string expr) + => Northwind().ExecuteQuery($"SELECT {expr} FROM Employees WHERE EmployeeID = 1").ColumnTypes[0]; + + [Fact] + public void Greatest_returns_the_largest_argument() + => Assert.Equal(9, Convert.ToInt32(Scalar("GREATEST(3, 9, -4, 7)"))); + + [Fact] + public void Least_returns_the_smallest_argument() + => Assert.Equal(-4, Convert.ToInt32(Scalar("LEAST(3, 9, -4, 7)"))); + + [Fact] + public void Null_arguments_are_ignored() + { + // SQL Server's and PostgreSQL's rule, and the one EF Core translates Math.Max/Min against: a NULL does + // not make the answer NULL while another argument has a value. + Assert.Equal(5, Convert.ToInt32(Scalar("GREATEST(NULL, 5, NULL, 2)"))); + Assert.Equal(2, Convert.ToInt32(Scalar("LEAST(NULL, 5, NULL, 2)"))); + } + + [Fact] + public void All_null_arguments_yield_null() + { + Assert.Null(Scalar("GREATEST(NULL, NULL)")); + Assert.Null(Scalar("LEAST(NULL, NULL)")); + } + + [Fact] + public void A_single_argument_is_accepted() + { + Assert.Equal(4, Convert.ToInt32(Scalar("GREATEST(4)"))); + Assert.Equal(4, Convert.ToInt32(Scalar("LEAST(4)"))); + } + + [Fact] + public void Zero_arguments_is_an_error() + { + Assert.ThrowsAny(() => Scalar("GREATEST()")); + Assert.ThrowsAny(() => Scalar("LEAST()")); + } + + [Fact] + public void Mixed_numeric_arguments_compare_by_value() + { + Assert.Equal(2.5, Convert.ToDouble(Scalar("GREATEST(2, 2.5E0, 1)"))); + Assert.Equal(1, Convert.ToDouble(Scalar("LEAST(2, 2.5E0, 1)"))); + } + + [Fact] + public void Text_compares_as_the_comparison_operators_do() + { + // Case-insensitively, as "=" and "<" compare text in Access's default "Compare Database" mode. + Assert.Equal("pear", Scalar("GREATEST('apple', 'pear', 'Banana')")); + Assert.Equal("apple", Scalar("LEAST('apple', 'pear', 'Banana')")); + } + + [Fact] + public void Dates_compare_by_value() + { + Assert.Equal(new DateTime(2024, 5, 1), Scalar("GREATEST(#2020-01-01#, #2024-05-01#, #2019-12-31#)")); + Assert.Equal(new DateTime(2019, 12, 31), Scalar("LEAST(#2020-01-01#, #2024-05-01#, #2019-12-31#)")); + } + + [Fact] + public void Works_over_columns_in_a_where_clause() + { + var e = Northwind(); + int[] got = e.ExecuteQuery( + "SELECT OrderID FROM [Order Details] WHERE GREATEST(Quantity, 100) = Quantity ORDER BY OrderID") + .Rows.Select(r => Convert.ToInt32(r[0])).ToArray(); + int[] expected = e.ExecuteQuery( + "SELECT OrderID FROM [Order Details] WHERE Quantity >= 100 ORDER BY OrderID") + .Rows.Select(r => Convert.ToInt32(r[0])).ToArray(); + + Assert.NotEmpty(expected); + Assert.Equal(expected, got); + } + + [Fact] + public void Nests_and_composes_like_any_expression() + => Assert.Equal(6, Convert.ToInt32(Scalar("1 + GREATEST(LEAST(8, 5), 2)"))); + + [Theory] + [InlineData("greatest(1, 2)", 2)] + [InlineData("Least(1, 2)", 1)] + public void Name_is_case_insensitive(string expr, int expected) + => Assert.Equal(expected, Convert.ToInt32(Scalar(expr))); + + // Each returns one of its arguments, so each declares the type its arguments unify to, as COALESCE does. + + [Fact] + public void Uniform_argument_types_declare_that_type() + => Assert.Equal(typeof(string), ColumnType("GREATEST('a', 'b')")); + + [Fact] + public void A_null_argument_does_not_erase_the_declared_type() + => Assert.Equal(typeof(string), ColumnType("LEAST(NULL, 'a')")); + + [Fact] + public void Numeric_arguments_widen_to_the_larger_type() + => Assert.Equal(typeof(double), ColumnType("GREATEST(1, 2.5E0)")); +} diff --git a/test/LibRed.Engine.Tests/GroupedWindowTests.cs b/test/LibRed.Engine.Tests/GroupedWindowTests.cs new file mode 100644 index 000000000..3161efe3f --- /dev/null +++ b/test/LibRed.Engine.Tests/GroupedWindowTests.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// Seven sales by five reps in three regions — the data the grouped-window, FILTER and LISTAGG tests share. +/// Totals by rep: ann 150 and bob 200 in N, cat 300 and dan 30 in S, eve 70 in E; 750 in all. +internal static class Sales +{ + public static readonly string[] Setup = + [ + "CREATE TABLE S (Id LONG, Region TEXT(10), Rep TEXT(10), Amount LONG)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (1, 'N', 'ann', 100)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (2, 'N', 'ann', 50)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (3, 'N', 'bob', 200)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (4, 'S', 'cat', 300)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (5, 'S', 'dan', 10)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (6, 'S', 'dan', 20)", + "INSERT INTO S (Id, Region, Rep, Amount) VALUES (7, 'E', 'eve', 70)", + ]; + + /// Every row of 's result, its columns joined by ':' and the rows by ' '. + public static string Rows(SharedDatabase database, string sql) => string.Join(" ", + database.Query(sql, CultureInfo.InvariantCulture).Rows + .Select(row => string.Join(":", row.Select(v => Convert.ToString(v, CultureInfo.InvariantCulture))))); +} + +/// +/// Window functions over a grouped query: they run over the groups HAVING keeps, as the standard orders it, so they +/// can rank, total and compare groups by their aggregates. Access has no window functions; this is a LibRed +/// extension. +/// +public class GroupedWindowTests(GroupedWindowTests.Database database) + : TempDatabaseTest, IClassFixture +{ + public sealed class Database() : SharedDatabase("grouped-window-", Sales.Setup); + + private string Rows(string sql) => Sales.Rows(database, sql); + + [Fact] + public void Groups_rank_by_their_aggregate() => + Assert.Equal("ann:150:3 bob:200:2 cat:300:1 dan:30:5 eve:70:4", Rows( + "SELECT Rep, SUM(Amount), RANK() OVER (ORDER BY SUM(Amount) DESC) FROM S GROUP BY Rep ORDER BY Rep")); + + [Fact] + public void Groups_rank_within_a_partition_of_groups() => + Assert.Equal("E:eve:1 N:ann:2 N:bob:1 S:cat:1 S:dan:2", Rows( + "SELECT Region, Rep, RANK() OVER (PARTITION BY Region ORDER BY SUM(Amount) DESC) FROM S " + + "GROUP BY Region, Rep ORDER BY Region, Rep")); + + [Fact] + public void An_aggregate_of_an_aggregate_totals_the_groups() => + Assert.Equal("ann:150:750:150 bob:200:750:350 cat:300:750:650 dan:30:750:680 eve:70:750:750", Rows( + "SELECT Rep, SUM(Amount), SUM(SUM(Amount)) OVER (), SUM(SUM(Amount)) OVER (ORDER BY Rep) FROM S GROUP BY Rep ORDER BY Rep")); + + // The total is over the groups HAVING keeps — 150 + 300 + 30 = 480 — not all 750. + [Fact] + public void The_share_of_the_total_is_a_window_over_the_groups() => + Assert.Equal("ann:31.25 cat:62.5 dan:6.25", Rows( + "SELECT Rep, SUM(Amount) * 100 / SUM(SUM(Amount)) OVER () FROM S GROUP BY Rep " + + "HAVING Rep IN ('ann', 'cat', 'dan') ORDER BY Rep")); + + [Fact] + public void Having_runs_before_the_window() => + Assert.Equal("ann:1 bob:2 cat:3 eve:4", Rows( + "SELECT Rep, ROW_NUMBER() OVER (ORDER BY Rep) FROM S GROUP BY Rep HAVING SUM(Amount) > 60 ORDER BY Rep")); + + [Fact] + public void Order_by_can_sort_on_a_window_over_the_groups() => + Assert.Equal("dan eve ann bob cat", Rows( + "SELECT Rep FROM S GROUP BY Rep ORDER BY RANK() OVER (ORDER BY SUM(Amount))")); + + [Fact] + public void A_window_can_read_the_group_key() => + Assert.Equal("E: N:E S:N", Rows("SELECT Region, LAG(Region) OVER (ORDER BY Region) FROM S GROUP BY Region")); + + [Fact] + public void An_aggregate_in_a_window_alone_makes_the_query_one_group() => + Assert.Equal("1:750:1", Rows("SELECT RANK() OVER (ORDER BY SUM(Amount)), SUM(Amount), COUNT(*) OVER () FROM S")); + + [Fact] + public void The_window_columns_are_typed() + { + var (types, rows) = database.Query( + "SELECT RANK() OVER (ORDER BY SUM(Amount)), SUM(SUM(Amount)) OVER (), AVG(SUM(Amount)) OVER () FROM S GROUP BY Rep", + CultureInfo.InvariantCulture); + Assert.Equal([typeof(int), typeof(int), typeof(double)], types); + Assert.All(rows, row => Assert.Equal(150.0, row[2])); + } +} + +/// +/// An aggregate's FILTER (WHERE …): only the rows the condition is true for go in — grouped, in HAVING, over a +/// window and in a correlated subquery. Access has no FILTER; this is a LibRed extension. +/// +public class AggregateFilterTests(AggregateFilterTests.Database database) + : TempDatabaseTest, IClassFixture +{ + public sealed class Database() : SharedDatabase("aggregate-filter-", Sales.Setup); + + private string Rows(string sql) => Sales.Rows(database, sql); + + [Theory] + [InlineData("COUNT(*) FILTER (WHERE Amount > 60)", "E:1 N:2 S:1")] + [InlineData("SUM(Amount) FILTER (WHERE Rep = 'dan')", "E: N: S:30")] + [InlineData("MAX(Rep) FILTER (WHERE Amount < 100)", "E:eve N:ann S:dan")] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY Amount) FILTER (WHERE Amount > 20)", "E:70 N:100 S:300")] + public void A_filter_narrows_each_group(string aggregate, string expected) => + Assert.Equal(expected, Rows($"SELECT Region, {aggregate} FROM S GROUP BY Region ORDER BY Region")); + + [Fact] + public void A_filtered_count_of_everything_counts_the_filtered_rows() => + Assert.Equal("0:7", Rows("SELECT COUNT(*) FILTER (WHERE Amount > 1000), COUNT(*) FROM S")); + + [Fact] + public void Having_can_test_a_filtered_aggregate() => + Assert.Equal("N", Rows("SELECT Region FROM S GROUP BY Region HAVING COUNT(*) FILTER (WHERE Amount > 60) >= 2")); + + [Theory] + [InlineData("SUM(Amount) FILTER (WHERE Amount >= 100) OVER (ORDER BY Id)", "1:100 2:100 3:300 4:600 5:600 6:600 7:600")] + [InlineData("COUNT(*) FILTER (WHERE Region = 'S') OVER (PARTITION BY Region)", "1:0 2:0 3:0 4:3 5:3 6:3 7:0")] + public void A_filter_narrows_each_frame(string expression, string expected) => + Assert.Equal(expected, Rows($"SELECT Id, {expression} FROM S ORDER BY Id")); + + [Fact] + public void A_filter_can_read_the_outer_query() => + Assert.Equal("1:2 2:4 3:1 4:0 5:6 6:5 7:3", Rows( + "SELECT s1.Id, (SELECT COUNT(*) FILTER (WHERE s2.Amount > s1.Amount) FROM S AS s2) FROM S AS s1 ORDER BY s1.Id")); + + [Theory] + [InlineData("SELECT UCASE(Rep) FILTER (WHERE Id > 1) FROM S")] + [InlineData("SELECT FIRST_VALUE(Amount) FILTER (WHERE Id > 1) OVER () FROM S")] + [InlineData("SELECT ROW_NUMBER() FILTER (WHERE Id > 1) OVER (ORDER BY Id) FROM S")] + public void Only_an_aggregate_takes_a_filter(string sql) => + Assert.Throws(() => Rows(sql)); +} diff --git a/test/LibRed.Engine.Tests/IndexSelectionRefusalTests.cs b/test/LibRed.Engine.Tests/IndexSelectionRefusalTests.cs index d80981254..bf4058a3d 100644 --- a/test/LibRed.Engine.Tests/IndexSelectionRefusalTests.cs +++ b/test/LibRed.Engine.Tests/IndexSelectionRefusalTests.cs @@ -43,9 +43,15 @@ public void Fully_constraining_the_composite_index_uses_one_point_seek() [Theory] [InlineData("SELECT Id FROM T WHERE A > (SELECT MAX(A) FROM T)")] [InlineData("SELECT Id FROM T WHERE A = B")] + [InlineData("SELECT Id FROM T WHERE Id > CASE WHEN A = 10 THEN 0 ELSE 5 END")] + [InlineData("SELECT Id FROM T WHERE Id BETWEEN A AND 5")] public void A_row_or_subquery_dependent_value_is_not_a_seek_bound(string sql) => Assert.False(ContainsSeek(Fresh().PlanFor(sql))); + [Fact] + public void A_case_over_the_row_is_evaluated_per_row() + => Assert.Single(Fresh().ExecuteQuery("SELECT Id FROM T WHERE Id > CASE WHEN A = 10 THEN 0 ELSE 5 END").Rows); + [Fact] public void A_computed_derived_projection_is_not_assumed_hash_compatible() { diff --git a/test/LibRed.Engine.Tests/InstrRevEdgeCasesTests.cs b/test/LibRed.Engine.Tests/InstrRevEdgeCasesTests.cs index 4c2665b86..d8a8eebd2 100644 --- a/test/LibRed.Engine.Tests/InstrRevEdgeCasesTests.cs +++ b/test/LibRed.Engine.Tests/InstrRevEdgeCasesTests.cs @@ -41,7 +41,10 @@ public void InstrRev_matches_ace(string expr, int expected) [Theory] [InlineData("InStrRev(Null, 'a')")] // Data type mismatch [InlineData("InStrRev('abc', Null)")] // Data type mismatch - [InlineData("InStrRev('abcabc', 'a', 0)")] // Invalid procedure call (start must be -1 or >= 1) public void InstrRev_error_cases(string expr) => Assert.Throws(() => Eval(expr)); + + [Fact] + public void InstrRev_start_of_zero_is_an_invalid_procedure_call() // start must be -1 or >= 1 + => Assert.Throws(() => Eval("InStrRev('abcabc', 'a', 0)")); } diff --git a/test/LibRed.Engine.Tests/IntegerOperatorRangeTests.cs b/test/LibRed.Engine.Tests/IntegerOperatorRangeTests.cs new file mode 100644 index 000000000..68c0019cd --- /dev/null +++ b/test/LibRed.Engine.Tests/IntegerOperatorRangeTests.cs @@ -0,0 +1,61 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// MOD and \ over a Double, Decimal or Currency past a Long. ACE squeezes each operand into a Long +/// first and overflows; LibRed works the answer out in Int64 and needs only the result to fit its type — Int32 +/// unless an operand is an Int64 — so a remainder, always below its divisor, always does. A LibRed extension. +/// +public class IntegerOperatorRangeTests(IntegerOperatorRangeTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE R (K LONG, D DOUBLE, M DECIMAL(20,0), Y CURRENCY, I LONG, L BIGINT)", + "INSERT INTO R (K, D, M, Y, I, L) VALUES (1, 1000000000000, 1000000000000, 922337203685477, 7, 1000000000000)", + ]; + + public sealed class Database() : SharedDatabase("integer-range-", Setup); + + private (Type Declared, object? Value) Query(string expression) + { + var (types, rows) = database.Query($"SELECT {expression} FROM R", CultureInfo.InvariantCulture); + return (types[0], rows.Single()[0]); + } + + [Theory] + [InlineData("D MOD 7", 1)] + [InlineData("M MOD 7", 1)] + [InlineData("Y MOD 7", (int)(922337203685477L % 7))] + [InlineData("D MOD I", 1)] + [InlineData("7 MOD D", 7)] // the divisor past a Long, too + [InlineData("-D MOD 7", -1)] // the remainder takes the dividend's sign + [InlineData("D \\ 1000000", 1000000)] // a quotient that fits + [InlineData("Y \\ 1000000000", 922337)] + public void The_answer_is_an_int32_when_it_fits_one(string expression, int expected) + { + var (declared, value) = Query(expression); + Assert.Equal(typeof(int), declared); + Assert.Equal(expected, value); + } + + // The column's type is settled before any value is read, so a quotient past a Long cannot become an Int64. + [Theory] + [InlineData("D \\ 7")] + [InlineData("M \\ 2")] + public void A_quotient_past_a_long_is_an_overflow(string expression) => + Assert.Throws(() => Query(expression)); + + [Theory] + [InlineData("L MOD 7", 1L)] + [InlineData("D MOD 864000000000", 136000000000L)] + [InlineData("L \\ 7", 142857142857L)] + public void With_an_int64_operand_it_is_an_int64(string expression, long expected) + { + var (declared, value) = Query(expression); + Assert.Equal(typeof(long), declared); + Assert.Equal(expected, value); + } +} diff --git a/test/LibRed.Engine.Tests/KeptTypeFunctionDeclarationTests.cs b/test/LibRed.Engine.Tests/KeptTypeFunctionDeclarationTests.cs new file mode 100644 index 000000000..62e206f57 --- /dev/null +++ b/test/LibRed.Engine.Tests/KeptTypeFunctionDeclarationTests.cs @@ -0,0 +1,49 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Round, Abs, Int and Fix keep their operand's type, and are declared as they return: undeclared, a Round beside a +/// whole number in IIF, CASE or COALESCE let the whole number declare the column while the Round arm returned a +/// Decimal. Sgn is an Integer. +/// +public class KeptTypeFunctionDeclarationTests(KeptTypeFunctionDeclarationTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE K (Id LONG, Y CURRENCY, E DECIMAL(18,4), D DOUBLE, R REAL, S SMALLINT, L BIGINT, T DATETIME, X TEXT(10))", + "INSERT INTO K (Id, Y, E, D, R, S, L, T, X) VALUES (1, 2.5678, 2.5678, 2.5678, 2.5, -7, 5, #2020-01-02 12:00:00#, '2.5')", + ]; + + public sealed class Database() : SharedDatabase("kept-type-", Setup); + + [Theory] + [InlineData("ROUND(Y, 2)", typeof(decimal))] + [InlineData("ROUND(E)", typeof(decimal))] + [InlineData("ROUND(D, 1)", typeof(double))] + [InlineData("ROUND(R)", typeof(float))] + [InlineData("ROUND(S)", typeof(int))] + [InlineData("ROUND(L)", typeof(long))] + [InlineData("ROUND(X)", typeof(double))] + [InlineData("ROUND(T)", typeof(double))] + [InlineData("ABS(Y)", typeof(decimal))] + [InlineData("ABS(S)", typeof(int))] + [InlineData("ABS(T)", typeof(double))] + [InlineData("INT(D)", typeof(double))] + [InlineData("INT(T)", typeof(DateTime))] + [InlineData("FIX(Y)", typeof(decimal))] + [InlineData("FIX(T)", typeof(DateTime))] + [InlineData("SGN(Y)", typeof(int))] + [InlineData("IIF(Id = 1, ROUND(Y), 0)", typeof(decimal))] + [InlineData("IIF(Id = 2, ROUND(Y), 0)", typeof(decimal))] + [InlineData("CASE WHEN Id = 1 THEN ROUND(Y, 2) ELSE 5 END", typeof(decimal))] + [InlineData("COALESCE(ROUND(Y), S)", typeof(decimal))] + public void The_column_is_declared_as_the_value_it_returns(string expression, Type expected) + { + var (types, rows) = database.Query($"SELECT {expression} AS c FROM K", CultureInfo.InvariantCulture); + Assert.Equal(expected, types[0]); + Assert.IsType(expected, rows.Single()[0]); + } +} diff --git a/test/LibRed.Engine.Tests/LeftRightEdgeCasesTests.cs b/test/LibRed.Engine.Tests/LeftRightEdgeCasesTests.cs index 2de5eca28..81ffe3c48 100644 --- a/test/LibRed.Engine.Tests/LeftRightEdgeCasesTests.cs +++ b/test/LibRed.Engine.Tests/LeftRightEdgeCasesTests.cs @@ -37,13 +37,17 @@ public void Null_string_propagates(string expr) => Assert.Null(Eval(expr)); [Theory] - [InlineData("Left('abcdef', -1)")] // negative → Invalid procedure call - [InlineData("Right('abcdef', -1)")] [InlineData("Left('abc', Null)")] // null length → Data type mismatch [InlineData("Right('abc', Null)")] public void Error_cases(string expr) => Assert.Throws(() => Eval(expr)); + [Theory] + [InlineData("Left('abcdef', -1)")] // negative → Invalid procedure call + [InlineData("Right('abcdef', -1)")] + public void Negative_length_is_an_invalid_procedure_call(string expr) + => Assert.Throws(() => Eval(expr)); + [Fact] public void Split_is_not_a_scalar_function() // matches ACE ("Undefined function 'Split'") => Assert.Throws(() => Eval("Split('a,b,c', ',')")); diff --git a/test/LibRed.Engine.Tests/LikeTests.cs b/test/LibRed.Engine.Tests/LikeTests.cs index 21551d7ba..9c36b0e8b 100644 --- a/test/LibRed.Engine.Tests/LikeTests.cs +++ b/test/LibRed.Engine.Tests/LikeTests.cs @@ -4,9 +4,8 @@ namespace LibRed.Engine.Tests; -// Access/Jet LIKE wildcards, including the bracket char class [ ... ] / [! ... ] and the # digit wildcard. -// EF escapes literal special chars by bracketing them (Contains("C#") -> LIKE '%C[#]%'), so [#] must match -// a literal '#', not the three characters "[#]". +// ANSI-92 LIKE wildcards, including the bracket char class [ ... ] / [! ... ]. EF escapes special chars by +// bracketing them (Contains("C#") -> LIKE '%C[#]%'), so [#] must match a literal '#', not the three characters "[#]". public class LikeTests : TempDatabaseTest { private static QueryEngine Fresh(params string[] values) @@ -33,10 +32,12 @@ public void Bracketed_hash_matches_a_literal_hash() } [Fact] - public void Hash_is_a_digit_wildcard_when_not_bracketed() + public void Hash_star_and_question_mark_are_plain_characters() { - var e = Fresh("A5", "AB", "A0", "A"); - Assert.Equal(["A5", "A0"], Match(e, "A#")); + var e = Fresh("A5", "A#", "A*", "A?", "AB"); + Assert.Equal(["A#"], Match(e, "A#")); + Assert.Equal(["A*"], Match(e, "A*")); + Assert.Equal(["A?"], Match(e, "A?")); } [Fact] diff --git a/test/LibRed.Engine.Tests/ListAggTests.cs b/test/LibRed.Engine.Tests/ListAggTests.cs new file mode 100644 index 000000000..81d241950 --- /dev/null +++ b/test/LibRed.Engine.Tests/ListAggTests.cs @@ -0,0 +1,64 @@ +using System.Globalization; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The standard's LISTAGG(x [, separator]) WITHIN GROUP (ORDER BY …), grouped and over a window. Access has no +/// such aggregate; this is a LibRed extension. +/// +public class ListAggTests(ListAggTests.Database database) + : TempDatabaseTest, IClassFixture +{ + public sealed class Database() : SharedDatabase("listagg-", Sales.Setup); + + private string Rows(string sql) => Sales.Rows(database, sql); + + [Theory] + [InlineData("LISTAGG(Rep, ', ') WITHIN GROUP (ORDER BY Rep)", "E:eve N:ann, ann, bob S:cat, dan, dan")] + [InlineData("LISTAGG(DISTINCT Rep, ',') WITHIN GROUP (ORDER BY Rep)", "E:eve N:ann,bob S:cat,dan")] + [InlineData("LISTAGG(Amount, '-') WITHIN GROUP (ORDER BY Rep DESC, Amount)", "E:70 N:200-50-100 S:10-20-300")] + [InlineData("LISTAGG(Rep, ',') WITHIN GROUP (ORDER BY Id) FILTER (WHERE Amount > 60)", "E:eve N:ann,bob S:cat")] + public void The_values_of_each_group_are_listed_in_order(string aggregate, string expected) => + Assert.Equal(expected, Rows($"SELECT Region, {aggregate} FROM S GROUP BY Region ORDER BY Region")); + + [Fact] + public void Without_a_separator_the_values_run_together() => + Assert.Equal("annannbobcatdandaneve", Rows("SELECT LISTAGG(Rep) WITHIN GROUP (ORDER BY Id) FROM S")); + + [Fact] + public void No_values_list_as_null() => + Assert.Equal("", Rows("SELECT LISTAGG(Rep, ',') WITHIN GROUP (ORDER BY Id) FROM S WHERE Id > 100")); + + [Fact] + public void Nulls_are_left_out() => + Assert.Equal("ann,bob", Rows( + "SELECT LISTAGG(IIF(Amount > 60 AND Region = 'N', Rep, NULL), ',') WITHIN GROUP (ORDER BY Id) FROM S")); + + [Fact] + public void The_column_is_text() + { + var (types, rows) = database.Query("SELECT LISTAGG(Id, ',') WITHIN GROUP (ORDER BY Id) FROM S", CultureInfo.InvariantCulture); + Assert.Equal(typeof(string), types[0]); + Assert.Equal("1,2,3,4,5,6,7", rows[0][0]); + } + + [Theory] + [InlineData("LISTAGG(Rep, ',') WITHIN GROUP (ORDER BY Id) OVER (PARTITION BY Region)", + "1:ann,ann,bob 2:ann,ann,bob 3:ann,ann,bob 4:cat,dan,dan 5:cat,dan,dan 6:cat,dan,dan 7:eve")] + [InlineData("LISTAGG(Id, ',') WITHIN GROUP (ORDER BY Id) OVER (ORDER BY Id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)", + "1:1 2:1,2 3:2,3 4:3,4 5:4,5 6:5,6 7:6,7")] + [InlineData("LISTAGG(DISTINCT Rep, '/') WITHIN GROUP (ORDER BY Rep DESC) OVER (ORDER BY Id)", + "1:ann 2:ann 3:bob/ann 4:cat/bob/ann 5:dan/cat/bob/ann 6:dan/cat/bob/ann 7:eve/dan/cat/bob/ann")] + public void Over_a_window_each_frame_is_listed(string expression, string expected) => + Assert.Equal(expected, Rows($"SELECT Id, {expression} FROM S ORDER BY Id")); + + [Theory] + [InlineData("LISTAGG(Rep, ',')")] + [InlineData("LISTAGG(Rep, Region) WITHIN GROUP (ORDER BY Id)")] + [InlineData("LISTAGG(Rep, ',', 'x') WITHIN GROUP (ORDER BY Id)")] + [InlineData("LISTAGG(*) WITHIN GROUP (ORDER BY Id)")] + public void The_syntax_is_the_standards(string aggregate) => + Assert.Throws(() => Rows($"SELECT {aggregate} FROM S")); +} diff --git a/test/LibRed.Engine.Tests/LogicalOperatorTests.cs b/test/LibRed.Engine.Tests/LogicalOperatorTests.cs new file mode 100644 index 000000000..f5a7a62ff --- /dev/null +++ b/test/LibRed.Engine.Tests/LogicalOperatorTests.cs @@ -0,0 +1,118 @@ +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The logical operators NOT AND OR XOR EQV IMP and a WHERE +/// condition: every value is False when it reads as 0 and True otherwise, Null follows the VBA truth tables, and the +/// operators bind in that order. The expected values were measured against ACE. +/// +public class LogicalOperatorTests(LogicalOperatorTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, TN TEXT(60), NT TEXT(60), D DATETIME, G GUID, B BINARY(4))", + "INSERT INTO T (Id, TN, D) VALUES (1, '7', #2020-01-02 12:00:00#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("logic-ops-", Setup); + + private object? Query(string sql) => database.Engine.ExecuteQuery(sql).Rows.First()[0]; + + private object? Scalar(string expression) => Query($"SELECT {expression} FROM T"); + + [Theory] + [InlineData("NOT 10", false)] + [InlineData("NOT 0", true)] + [InlineData("NOT 0.5", false)] + [InlineData("NOT '0'", true)] + [InlineData("NOT '1'", false)] + [InlineData("NOT 'abc'", false)] + [InlineData("NOT ''", false)] + [InlineData("NOT 'True'", false)] + [InlineData("NOT TN", false)] + [InlineData("NOT D", false)] + [InlineData("NOT G", false)] + [InlineData("NOT B", false)] + [InlineData("NOT LEFT('1', 1)", false)] + [InlineData("NOT UCASE('abc')", false)] + public void A_value_is_false_only_when_it_reads_as_zero(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("12 AND 10", true)] + [InlineData("10 AND '0'", false)] + [InlineData("10 AND 'abc'", true)] + [InlineData("10 AND ''", true)] + [InlineData("10 AND G", true)] + [InlineData("D AND 1", true)] + [InlineData("0 AND NULL", false)] + [InlineData("NULL AND 0", false)] + [InlineData("12 OR 10", true)] + [InlineData("0 OR '0'", false)] + [InlineData("1 OR NULL", true)] + [InlineData("NULL OR -1", true)] + [InlineData("12 XOR 10", false)] + [InlineData("FALSE XOR TRUE", true)] + [InlineData("'0' XOR 'abc'", true)] + [InlineData("12 EQV 10", true)] + [InlineData("TRUE EQV FALSE", false)] + [InlineData("FALSE EQV '0'", true)] + [InlineData("TRUE IMP FALSE", false)] + [InlineData("FALSE IMP FALSE", true)] + [InlineData("0 IMP NULL", true)] + [InlineData("NULL IMP -1", true)] + [InlineData("NULL IMP 'abc'", true)] + public void The_operators_are_logical_not_bitwise(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NOT NULL")] + [InlineData("NOT NT")] + [InlineData("1 AND NULL")] + [InlineData("NULL AND 0.5")] + [InlineData("0 OR NULL")] + [InlineData("NULL XOR 0")] + [InlineData("NULL EQV -1")] + [InlineData("-1 IMP NULL")] + [InlineData("NULL IMP 0")] + [InlineData("NULL IMP '0'")] + [InlineData("NULL IMP NULL")] + public void Null_follows_the_truth_tables(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("1 = 1 OR 1 = 2 AND 1 = 2", true)] + [InlineData("(1 = 1 OR 1 = 2) AND 1 = 2", false)] + [InlineData("NOT 1 = 1 OR 1 = 1", true)] + [InlineData("NOT TRUE AND FALSE", false)] + [InlineData("TRUE XOR TRUE OR TRUE", false)] + [InlineData("TRUE OR TRUE XOR TRUE", false)] + [InlineData("TRUE EQV FALSE XOR TRUE", true)] + [InlineData("FALSE IMP FALSE EQV FALSE", true)] + [InlineData("TRUE IMP FALSE IMP FALSE", true)] + [InlineData("10 AND 8 = 8", true)] + [InlineData("(10 AND 8) = 8", false)] + public void Not_and_or_xor_eqv_imp_bind_in_that_order_left_to_right(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'abc'", 1)] + [InlineData("''", 1)] + [InlineData("'0'", 0)] + [InlineData("TN", 1)] + [InlineData("D", 1)] + [InlineData("G", 1)] + [InlineData("NT", 0)] + [InlineData("NOT '0'", 1)] + [InlineData("NOT NULL", 0)] + [InlineData("1 AND NULL", 0)] + [InlineData("NULL IMP -1", 1)] + [InlineData("4 IMP NULL", 0)] + public void A_where_condition_uses_the_same_truth(string condition, int expected) => + Assert.Equal(expected, Query($"SELECT COUNT(*) FROM T WHERE {condition}")); +} diff --git a/test/LibRed.Engine.Tests/LvalReclamationTests.cs b/test/LibRed.Engine.Tests/LvalReclamationTests.cs index f854e0cd2..4fdb57aef 100644 --- a/test/LibRed.Engine.Tests/LvalReclamationTests.cs +++ b/test/LibRed.Engine.Tests/LvalReclamationTests.cs @@ -98,19 +98,29 @@ public void Deleting_a_row_reclaims_its_memo_pages() } afterInserts = new FileInfo(path).Length; - using (var db = JetDatabase.Open(path, readOnly: false)) + // A deleted row's memo pages are held until the connection closes, as ACE holds them: the first + // session's churn grows the file, and the second reuses what the first released at close. Measured + // against ACE with the same loop: +209 pages, then +0. + void Churn() { + using var db = JetDatabase.Open(path, readOnly: false); var e = new QueryEngine(db); for (int i = 0; i < 20; i++) { - e.ExecuteNonQuery("DELETE FROM T"); // frees the memo pages - e.ExecuteNonQuery($"INSERT INTO T (M) VALUES ('{Big((char)('b' + i % 20))}')"); // reuses them + e.ExecuteNonQuery("DELETE FROM T"); // releases the memo pages + e.ExecuteNonQuery($"INSERT INTO T (M) VALUES ('{Big((char)('b' + i % 20))}')"); } } - long afterChurn = new FileInfo(path).Length; - Assert.True(afterChurn - afterInserts < 150_000, - $"file grew {afterChurn - afterInserts} bytes over 20 delete+insert cycles — deleted memo pages not reclaimed?"); + Churn(); + long afterFirstChurn = new FileInfo(path).Length; + Assert.True(afterFirstChurn - afterInserts > 10 * 20_000, + $"file grew only {afterFirstChurn - afterInserts} bytes over 20 delete+insert cycles in one session — deleted memo pages reused before close?"); + + Churn(); + long afterSecondChurn = new FileInfo(path).Length; + Assert.True(afterSecondChurn - afterFirstChurn < 150_000, + $"file grew {afterSecondChurn - afterFirstChurn} bytes over a second session's churn — memo pages not released at close?"); } finally { TemporaryDatabase.Delete(path); } } diff --git a/test/LibRed.Engine.Tests/MidReplaceEdgeCasesTests.cs b/test/LibRed.Engine.Tests/MidReplaceEdgeCasesTests.cs index 1f409ce8c..b7ef05a5d 100644 --- a/test/LibRed.Engine.Tests/MidReplaceEdgeCasesTests.cs +++ b/test/LibRed.Engine.Tests/MidReplaceEdgeCasesTests.cs @@ -60,5 +60,5 @@ public void Replace_null_string_argument_propagates(string expr) [InlineData("Mid('abcdef', 3, -1)")] // negative length → Invalid procedure call [InlineData("Replace('abcabc', 'b', 'X', 0)")] // start < 1 → Invalid procedure call public void Error_cases(string expr) - => Assert.Throws(() => Eval(expr)); + => Assert.Throws(() => Eval(expr)); } diff --git a/test/LibRed.Engine.Tests/MultiTableUpdateDeleteTests.cs b/test/LibRed.Engine.Tests/MultiTableUpdateDeleteTests.cs index fe6de0f96..d16e1a56b 100644 --- a/test/LibRed.Engine.Tests/MultiTableUpdateDeleteTests.cs +++ b/test/LibRed.Engine.Tests/MultiTableUpdateDeleteTests.cs @@ -52,6 +52,86 @@ public void Multi_table_update_touches_both_tables_and_accumulates_on_the_one_si finally { TemporaryDatabase.Delete(path); } } + // Every SET reads the row as it was before any of them (verified vs ACE), in one table and across joined ones. + [Theory] + [InlineData("UPDATE S SET X = X + 5, Y = X", "6|1")] + [InlineData("UPDATE S SET X = Y, Y = X", "2|1")] + [InlineData("UPDATE S SET Y = X, X = Y + 10", "12|1")] + public void Every_set_reads_the_original_row(string update, string expected) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = new QueryEngine(db); + e.ExecuteNonQuery("CREATE TABLE S (X long, Y long)"); + e.ExecuteNonQuery("INSERT INTO S (X, Y) VALUES (1, 2)"); + + e.ExecuteNonQuery(update); + + object?[] row = e.ExecuteQuery("SELECT X, Y FROM S").Rows.Single(); + Assert.Equal(expected, $"{row[0]}|{row[1]}"); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void A_set_on_one_joined_table_reads_the_other_before_its_set() + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = Seed(db); + + e.ExecuteNonQuery("UPDATE P INNER JOIN C ON P.Id = C.ParentId SET C.CName = P.PName, P.PName = C.CName WHERE C.Id = 12"); + + Assert.Equal("c12", e.ExecuteQuery("SELECT PName FROM P WHERE Id = 2").Rows.Single()[0]); + Assert.Equal("p2", e.ExecuteQuery("SELECT CName FROM C WHERE Id = 12").Rows.Single()[0]); + } + finally { TemporaryDatabase.Delete(path); } + } + + // UPDATE takes a comma list of tables, as FROM does, joined or not (verified vs ACE); the row count is the + // number of joined rows. + [Theory] + [InlineData("UPDATE P, C SET P.Hits = C.Id WHERE P.Id = C.ParentId AND C.Id <> 11", 2, "10|12")] + [InlineData("UPDATE P, C SET P.Hits = 7", 6, "7|7")] + [InlineData("UPDATE P, C INNER JOIN P AS Q ON C.ParentId = Q.Id SET P.Hits = C.Id WHERE P.Id = Q.Id AND C.Id <> 10", 2, "11|12")] + [InlineData("UPDATE P INNER JOIN (C INNER JOIN P AS Q ON C.ParentId = Q.Id) ON P.Id = Q.Id SET P.Hits = P.Hits + 1", 3, "2|1")] + public void Update_takes_a_comma_list_of_tables(string update, int affected, string hits) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = Seed(db); + + Assert.Equal(affected, e.ExecuteNonQuery(update)); + Assert.Equal(hits, string.Join("|", e.ExecuteQuery("SELECT Hits FROM P ORDER BY Id").Rows.Select(r => r[0]))); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Theory] + [InlineData("UPDATE P SET Hits = 1, Hits = 2")] + [InlineData("UPDATE P SET P.Hits = 1, hits = 2")] + [InlineData("UPDATE P INNER JOIN C ON P.Id = C.ParentId SET P.Hits = 1, P.Hits = 2")] + public void A_column_set_twice_is_an_error(string update) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = Seed(db); + + var error = Assert.Throws(() => e.ExecuteNonQuery(update)); + Assert.Contains("Duplicate output destination", error.Message); + Assert.Equal(0, Convert.ToInt32(e.ExecuteQuery("SELECT Hits FROM P WHERE Id = 1").Rows.Single()[0])); + } + finally { TemporaryDatabase.Delete(path); } + } + [Fact] public void Multi_table_delete_removes_only_the_targeted_table() { @@ -71,6 +151,232 @@ public void Multi_table_delete_removes_only_the_targeted_table() finally { TemporaryDatabase.Delete(path); } } + // Three tables where A and B share Ids 1 and 2, B and C share 4 and C has 1: A3, B4 and C4's partners are missing. + private static QueryEngine SeedThree(JetDatabase db) + { + var e = new QueryEngine(db); + e.ExecuteNonQuery("CREATE TABLE A (Id long, X long)"); + e.ExecuteNonQuery("CREATE TABLE B (K counter, Id long, V long, D long DEFAULT 9)"); + e.ExecuteNonQuery("CREATE TABLE C (Id long, W long)"); + foreach (int id in (int[])[1, 2, 3]) + e.ExecuteNonQuery($"INSERT INTO A (Id, X) VALUES ({id}, {id * 10})"); + foreach (int id in (int[])[1, 2, 4]) + e.ExecuteNonQuery($"INSERT INTO B (Id, V) VALUES ({id}, 0)"); + foreach (int id in (int[])[1, 4]) + e.ExecuteNonQuery($"INSERT INTO C (Id, W) VALUES ({id}, 0)"); + return e; + } + + private static string Rows(QueryEngine e, string sql) => + string.Join("|", e.ExecuteQuery(sql).Rows.Select(r => string.Join("/", r))); + + // A SET on the side of an outer join with no matching row writes a new row there, one per joined row, with the + // SET values and the table's defaults and AutoNumber (verified vs ACE). + [Fact] + public void A_set_on_the_unmatched_side_of_an_outer_join_adds_a_row() + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Equal(3, e.ExecuteNonQuery("UPDATE A LEFT JOIN B ON A.Id = B.Id SET A.X = 1, B.V = A.X + 100")); + Assert.Equal("1/1/110/9|2/2/120/9|3/4/0/9|4//130/9", Rows(e, "SELECT K, Id, V, D FROM B ORDER BY K")); + Assert.Equal("1/1|2/1|3/1", Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + + // Null values still make a row: B's rows with Id 4 and Null have no A. + Assert.Equal(4, e.ExecuteNonQuery("UPDATE A RIGHT JOIN B ON A.Id = B.Id SET A.X = NULL")); + Assert.Equal("/|/|1/|2/|3/1", Rows(e, "SELECT Id, X FROM A ORDER BY Id, X")); + Assert.Equal(5, e.ExecuteNonQuery("UPDATE A LEFT JOIN C ON A.Id = C.Id SET C.W = NULL")); + Assert.Equal("/|/|/|/|1/", Rows(e, "SELECT Id, W FROM C WHERE W IS NULL ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void A_new_row_that_breaks_a_rule_fails_the_whole_update() + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + e.ExecuteNonQuery("CREATE TABLE R (Id long, V long, Req long NOT NULL)"); + + Assert.Throws(() => + e.ExecuteNonQuery("UPDATE A LEFT JOIN R ON A.Id = R.Id SET A.X = 99, R.V = 3")); + Assert.Equal("1/10|2/20|3/30", Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + Assert.Equal("0", Rows(e, "SELECT COUNT(*) FROM R")); + } + finally { TemporaryDatabase.Delete(path); } + } + + // Joins onto a bracketed group, and DELETE over a comma list, in the shapes ACE accepts (verified vs ACE). A + // DELETE counts the joined rows where an outer join left the target without a row, as ACE does. + [Theory] + [InlineData("UPDATE A INNER JOIN (B LEFT JOIN C ON B.Id = C.Id) ON A.Id = B.Id SET A.X = 0", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE A LEFT JOIN (B LEFT JOIN C ON B.Id = C.Id) ON A.Id = B.Id SET A.X = 0", 3, "1/0|2/0|3/0")] + [InlineData("UPDATE A INNER JOIN (B INNER JOIN C ON B.Id = C.Id) ON A.Id = B.Id SET A.X = 0", 1, "1/0|2/20|3/30")] + [InlineData("UPDATE A RIGHT JOIN (B INNER JOIN C ON B.Id = C.Id) ON A.Id = B.Id SET A.X = 0", 2, "/0|1/0|2/20|3/30")] + [InlineData("UPDATE A, B LEFT JOIN C ON B.Id = C.Id SET A.X = 0 WHERE A.Id = B.Id", 2, "1/0|2/0|3/30")] + [InlineData("DELETE A.* FROM A INNER JOIN (B LEFT JOIN C ON B.Id = C.Id) ON A.Id = B.Id", 2, "3/30")] + [InlineData("DELETE A.* FROM A, B WHERE A.Id = B.Id", 2, "3/30")] + [InlineData("DELETE A.* FROM A, B INNER JOIN C ON B.Id = C.Id WHERE A.Id = B.Id", 1, "2/20|3/30")] + [InlineData("DELETE A.* FROM B LEFT JOIN A ON A.Id = B.Id", 3, "3/30")] + public void Joins_onto_a_group_and_comma_lists(string statement, int affected, string rows) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Equal(affected, e.ExecuteNonQuery(statement)); + Assert.Equal(rows, Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + // A derived table is written through to its tables' rows: the ones its join, WHERE, ORDER BY and TOP or OFFSET + // choose, under the names its projection gives the columns (verified vs ACE). An unqualified SET names the one + // table with that column, in a plain join too. + [Theory] + [InlineData("UPDATE (SELECT * FROM A INNER JOIN B ON A.Id = B.Id) SET X = 0", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE (SELECT TOP 1 * FROM A INNER JOIN B ON A.Id = B.Id ORDER BY A.Id DESC) AS Q SET Q.X = 0", 1, "1/10|2/0|3/30")] + [InlineData("UPDATE (SELECT A.X, B.V AS W FROM A INNER JOIN B ON A.Id = B.Id) AS Q SET Q.X = Q.W", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE (SELECT * FROM A INNER JOIN B ON A.Id = B.Id WHERE B.Id > 1) SET X = 0", 1, "1/10|2/0|3/30")] + [InlineData("UPDATE (SELECT A.Id, X FROM A INNER JOIN B ON A.Id = B.Id) AS Q INNER JOIN C ON Q.Id = C.Id SET Q.X = 0", 1, "1/0|2/20|3/30")] + [InlineData("UPDATE A INNER JOIN B ON A.Id = B.Id SET X = 0", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE (SELECT TOP 2 * FROM A ORDER BY Id DESC) SET X = 0", 2, "1/10|2/0|3/0")] + [InlineData("UPDATE (SELECT TOP 2 * FROM A ORDER BY Id DESC) AS Q SET Q.X = Q.Id", 2, "1/10|2/2|3/3")] + [InlineData("UPDATE (SELECT * FROM A AS T WHERE T.X > 10) SET X = 0", 2, "1/10|2/0|3/0")] + [InlineData("UPDATE (SELECT TOP 50 PERCENT * FROM A ORDER BY Id) SET X = 0", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE (SELECT * FROM A ORDER BY Id DESC OFFSET 1 ROWS) SET X = 0", 2, "1/0|2/0|3/30")] + [InlineData("UPDATE (SELECT X AS Y, Id FROM A) SET Y = Id", 3, "1/1|2/2|3/3")] + [InlineData("UPDATE (SELECT Id, X + 1 AS Z, X FROM A) SET X = 0", 3, "1/0|2/0|3/0")] + [InlineData("UPDATE (SELECT Id, X FROM A WHERE Id > 1) AS Q INNER JOIN B ON Q.Id = B.Id SET Q.X = 5", 1, "1/10|2/5|3/30")] + [InlineData("DELETE * FROM (SELECT TOP 2 * FROM A ORDER BY Id DESC)", 2, "1/10")] + [InlineData("DELETE Q.* FROM (SELECT TOP 1 * FROM A ORDER BY Id) AS Q", 1, "2/20|3/30")] + public void A_derived_table_is_written_through(string statement, int affected, string rows) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Equal(affected, e.ExecuteNonQuery(statement)); + Assert.Equal(rows, Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Theory] + [InlineData("UPDATE (SELECT TOP 2 Id FROM A ORDER BY Id DESC) SET X = 0", typeof(InvalidOperationException))] + [InlineData("UPDATE (SELECT X AS Y, Id FROM A) SET X = 0", typeof(InvalidOperationException))] + [InlineData("UPDATE (SELECT Id FROM A) SET Id = X", typeof(InvalidOperationException))] + [InlineData("UPDATE (SELECT DISTINCT * FROM A) SET X = 0", typeof(NotSupportedException))] + [InlineData("UPDATE (SELECT Id, COUNT(*) AS N FROM A GROUP BY Id) SET Id = 0", typeof(NotSupportedException))] + // A name more than one table has, and a DELETE through a join (verified vs ACE). + [InlineData("UPDATE (SELECT * FROM A INNER JOIN B ON A.Id = B.Id) SET Id = 0", typeof(InvalidOperationException))] + [InlineData("UPDATE A INNER JOIN B ON A.Id = B.Id SET Id = 0", typeof(InvalidOperationException))] + [InlineData("DELETE * FROM (SELECT * FROM A INNER JOIN B ON A.Id = B.Id)", typeof(InvalidOperationException))] + [InlineData("DELETE Q.* FROM (SELECT A.* FROM A INNER JOIN B ON A.Id = B.Id) AS Q", typeof(InvalidOperationException))] + [InlineData("DELETE A.* FROM (SELECT * FROM A INNER JOIN B ON A.Id = B.Id)", typeof(InvalidOperationException))] + public void A_derived_table_that_cannot_be_written_is_refused(string statement, Type error) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Throws(error, () => e.ExecuteNonQuery(statement)); + Assert.Equal("1/10|2/20|3/30", Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Fact] + public void A_derived_join_writes_both_tables_and_adds_rows_for_its_unmatched_side() + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Equal(2, e.ExecuteNonQuery("UPDATE (SELECT * FROM A INNER JOIN B ON A.Id = B.Id) SET V = X, X = V")); + Assert.Equal("1/0|2/0|3/30", Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + Assert.Equal("1/10|2/20|4/0", Rows(e, "SELECT Id, V FROM B ORDER BY Id")); + + Assert.Equal(3, e.ExecuteNonQuery("UPDATE (SELECT * FROM A LEFT JOIN B ON A.Id = B.Id) SET V = 7")); + Assert.Equal("1/1/7|2/2/7|3/4/0|4//7", Rows(e, "SELECT K, Id, V FROM B ORDER BY K")); + } + finally { TemporaryDatabase.Delete(path); } + } + + // A derived join can sit anywhere in a bracketed group a table can, joined and null-extended as a whole (verified + // vs ACE). Q is A joined to B, so Ids 1 and 2; C has 1 and 4, and D has 1 and 3. + [Theory] + [InlineData("UPDATE C INNER JOIN ((SELECT A.Id, X, V FROM A INNER JOIN B ON A.Id = B.Id) AS Q INNER JOIN D ON Q.Id = D.Id) ON C.Id = Q.Id SET Q.X = 0", 1, "1/0|2/20|3/30")] + [InlineData("UPDATE C INNER JOIN ((SELECT A.Id, X, V FROM A INNER JOIN B ON A.Id = B.Id) AS Q LEFT JOIN D ON Q.Id = D.Id) ON C.Id = Q.Id SET Q.X = 0", 1, "1/0|2/20|3/30")] + [InlineData("UPDATE C LEFT JOIN ((SELECT A.Id, X, V FROM A INNER JOIN B ON A.Id = B.Id) AS Q LEFT JOIN D ON Q.Id = D.Id) ON C.Id = Q.Id SET Q.X = 0", 2, "/0|1/0|2/20|3/30")] + [InlineData("UPDATE C, (SELECT A.Id, X, V FROM A INNER JOIN B ON A.Id = B.Id) AS Q INNER JOIN D ON Q.Id = D.Id SET Q.X = 0 WHERE C.Id = Q.Id", 1, "1/0|2/20|3/30")] + [InlineData("UPDATE C INNER JOIN (D INNER JOIN (SELECT A.Id, X, V FROM A INNER JOIN B ON A.Id = B.Id) AS Q ON D.Id = Q.Id) ON C.Id = D.Id SET Q.X = 0", 1, "1/0|2/20|3/30")] + public void A_derived_join_can_sit_in_a_group(string statement, int affected, string rows) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + e.ExecuteNonQuery("CREATE TABLE D (Id long, Z long)"); + e.ExecuteNonQuery("INSERT INTO D (Id, Z) VALUES (1, 0)"); + e.ExecuteNonQuery("INSERT INTO D (Id, Z) VALUES (3, 0)"); + + Assert.Equal(affected, e.ExecuteNonQuery(statement)); + Assert.Equal(rows, Rows(e, "SELECT Id, X FROM A ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + // Inside a LEFT-joined group, a table has no row where the group's first table has none, whatever its own ON + // would match (verified vs ACE: 5 joined rows, and A3's row writes a new C row). + [Fact] + public void A_left_joined_group_is_null_extended_as_a_whole() + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Equal(5, e.ExecuteNonQuery( + "UPDATE A LEFT JOIN (B LEFT JOIN C ON B.V = C.W) ON A.Id = B.Id SET C.W = 5")); + Assert.Equal("/5|1/5|4/5", Rows(e, "SELECT Id, W FROM C ORDER BY Id")); + } + finally { TemporaryDatabase.Delete(path); } + } + + [Theory] + [InlineData("UPDATE A LEFT JOIN (B INNER JOIN C ON B.Id = C.Id) ON A.Id = B.Id SET A.X = 0")] + [InlineData("UPDATE (A INNER JOIN B ON A.Id = B.Id) RIGHT JOIN C ON B.Id = C.Id SET A.X = 0")] + [InlineData("DELETE A.* FROM A LEFT JOIN (B INNER JOIN C ON B.Id = C.Id) ON A.Id = B.Id")] + public void Shapes_ace_refuses_are_not_supported(string statement) + { + string path = Fresh(); + try + { + using var db = JetDatabase.Open(path, readOnly: false); + var e = SeedThree(db); + + Assert.Throws(() => e.ExecuteNonQuery(statement)); + } + finally { TemporaryDatabase.Delete(path); } + } + // `DELETE *` (bare star) is fine for a single table, but a join DELETE without a `table.*` target is // ambiguous — Access rejects it ("specify the table"), and so does LibRed. [Fact] diff --git a/test/LibRed.Engine.Tests/NumberTextFunctionTests.cs b/test/LibRed.Engine.Tests/NumberTextFunctionTests.cs new file mode 100644 index 000000000..23c8d1daa --- /dev/null +++ b/test/LibRed.Engine.Tests/NumberTextFunctionTests.cs @@ -0,0 +1,209 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Str Val Hex Oct StrConv. The expected values were measured against ACE under +/// en-AU, except that a Null argument gives Null where ACE raises an error. +/// +public class NumberTextFunctionTests(NumberTextFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, NT TEXT(60), DT DATETIME, G GUID, B BINARY(4))", + "INSERT INTO T (Id, DT) VALUES (1, #2020-01-02 12:00:00#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("numbertext-", Setup); + + // Dates are written in the regional format, so each query runs under en-AU whatever the machine's culture. .NET's + // en-AU short date is d/M/yyyy, where Windows' (and so ACE's) is d/MM/yyyy. + private object? Scalar(string expression) => + database.Scalar($"SELECT {expression} FROM T", CultureInfo.GetCultureInfo("en-AU")); + + [Theory] + [InlineData("STR(1)", " 1")] + [InlineData("STR(-1)", "-1")] + [InlineData("STR(TRUE)", "-1")] + [InlineData("STR(FALSE)", " 0")] + [InlineData("STR(0.5)", " .5")] + [InlineData("STR(-0.5)", "-.5")] + [InlineData("STR(CCUR(-0.25))", "-.25")] + [InlineData("STR(CCUR(1.5))", " 1.5")] + [InlineData("STR(CINT(-5))", "-5")] + [InlineData("STR(CDBL(1) / 3)", " .333333333333333")] + [InlineData("STR(CDBL(2) / 3 * 10 ^ 14)", " 66666666666666.7")] + [InlineData("STR(CDBL(2) / 3 / 10 ^ 5)", " 6.66666666666667E-06")] + [InlineData("STR(CDBL(1) / 10 ^ 15)", " .000000000000001")] + [InlineData("STR(CDBL(1) / 10 ^ 16)", " 1E-16")] + [InlineData("STR(CDBL(123456789012345))", " 123456789012345")] + [InlineData("STR(CDBL(1234567890123456))", " 1.23456789012346E+15")] + [InlineData("STR(CDBL(-1.5) / 10 ^ 20)", "-1.5E-20")] + [InlineData("STR(CDBL(5E-324))", " 4.94065645841247E-324")] + [InlineData("STR(CSNG(0.1))", " .1")] + [InlineData("STR(CSNG(1) / CSNG(10 ^ 8))", " 1E-08")] + [InlineData("STR(CSNG(2) / CSNG(3))", " .6666667")] + [InlineData("STR(CSNG(2) / CSNG(3) * CSNG(1000000))", " 666666.7")] + [InlineData("STR(CSNG(1E-5))", " .00001")] + [InlineData("STR(CSNG(1E-10))", " 1E-10")] + [InlineData("STR(CSNG(1234567))", " 1234567")] + [InlineData("STR(CSNG(12345678))", " 1.234568E+07")] + [InlineData("STR(CSNG(-1E+15))", "-1E+15")] + [InlineData("STR(CSNG(3.4E+38))", " 3.4E+38")] + [InlineData("STR('&HFF')", " 255")] + [InlineData("STR('$5')", " 5")] + [InlineData("STR('1,000')", " 1000")] + [InlineData("STR(#2020-01-02#)", "2/1/2020")] + [InlineData("STR(DT)", "2/1/2020 12:00:00 pm")] + public void Str_writes_a_period_and_a_sign_column(string expression, string expected) => + Assert.Equal(expected, Scalar(expression) as string, ignoreCase: true); + + [Theory] + [InlineData("VAL(' - 5')", -5)] + [InlineData("VAL('3 .1 4')", 3.14)] + [InlineData("VAL(CHR(13) & '12')", 12)] + [InlineData("VAL(CHR(11) & '12')", 0)] + [InlineData("VAL(CHR(160) & '12')", 0)] + [InlineData("VAL('1' & CHR(160) & '2')", 1)] + [InlineData("VAL('1' & CHR(0) & '2')", 1)] + [InlineData("VAL('1d2')", 100)] + [InlineData("VAL('1D2')", 100)] + [InlineData("VAL('1d')", 1)] + [InlineData("VAL('1.5e2.5')", 150)] + [InlineData("VAL('1e-400')", 0)] + [InlineData("VAL('&HFF')", 255)] + [InlineData("VAL('&H 1F')", 31)] + [InlineData("VAL('&HFFg')", 255)] + [InlineData("VAL('&H7FFF')", 32767)] + [InlineData("VAL('&H8000')", -32768)] + [InlineData("VAL('&HFFFF')", -1)] + [InlineData("VAL('&H0000FFFF')", -1)] + [InlineData("VAL('&H10000')", 65536)] + [InlineData("VAL('&HFFFF1')", 1048561)] + [InlineData("VAL('&HFFFFFFFF')", -1)] + [InlineData("VAL('&H123456789')", 591751049)] + [InlineData("VAL('&HFFFFFFFFF')", -1)] + [InlineData("VAL('&O177777')", -1)] + [InlineData("VAL('&O100000')", -32768)] + [InlineData("VAL('&O77777777777')", -1)] + [InlineData("VAL('-&HFF')", 0)] + [InlineData("VAL('&B101')", 0)] + [InlineData("VAL(TRUE)", -1)] + [InlineData("VAL(G)", 0)] + [InlineData("VAL(DT)", 2)] + [InlineData("VAL(0.1)", 0.1)] + public void Val_reads_the_number_at_the_start(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)), 12); + + [Theory] + [InlineData("HEX(TRUE)", "FFFF")] + [InlineData("OCT(TRUE)", "177777")] + [InlineData("HEX(CINT(-1))", "FFFF")] + [InlineData("OCT(CINT(-1))", "177777")] + [InlineData("HEX(-1)", "FFFFFFFF")] + [InlineData("OCT(-1)", "37777777777")] + [InlineData("HEX(-32769)", "FFFF7FFF")] + [InlineData("HEX(CBYTE(255))", "FF")] + [InlineData("HEX(CCUR(-1))", "FFFFFFFFFFFFFFFF")] + [InlineData("HEX(CSNG(-1))", "FFFFFFFFFFFFFFFF")] + [InlineData("HEX('-1')", "FFFFFFFFFFFFFFFF")] + [InlineData("HEX('-1.5')", "FFFFFFFFFFFFFFFE")] + [InlineData("HEX('2.5')", "2")] + [InlineData("HEX(-0.5)", "0")] + [InlineData("HEX(4294967296)", "100000000")] + [InlineData("HEX(#2020-01-02#)", "AB38")] + [InlineData("OCT(DT)", "125470")] + [InlineData("HEX(-#2020-01-02#)", "FFFFFFFFFFFF54C8")] + [InlineData("HEX('&O17')", "F")] + [InlineData("OCT('1e3')", "1750")] + [InlineData("HEX('1,000')", "3E8")] + [InlineData("HEX('$5')", "5")] + public void Hex_and_oct_write_the_bits_of_the_type(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("STRCONV('hello wORLD', 0)", "hello wORLD")] + [InlineData("STRCONV(CHRW(454), 0)", "dž")] + [InlineData("STRCONV(TRUE, 1)", "-1")] + [InlineData("STRCONV(#2020-01-02#, 2)", "2/1/2020")] + [InlineData("STRCONV(B, 2)", "ab")] + [InlineData("STRCONV(B, 3)", "Ab")] + [InlineData("STRCONV('ß', 1)", "ß")] + [InlineData("STRCONV('İSTANBUL', 2)", "istanbul")] + [InlineData("STRCONV(CHRW(1044) & CHRW(1076), 2)", "??")] + [InlineData("STRCONV('dža', 3)", "?a")] + [InlineData("STRCONV('ABC' & CHRW(8364) & 'É', 2)", "abc€é")] + [InlineData("STRCONV('ÉCOLE élan', 3)", "École Élan")] + [InlineData("STRCONV('o''neil mc-donald', 3)", "O'neil Mc-donald")] + [InlineData("STRCONV('hello_world', 3)", "Hello_world")] + [InlineData("STRCONV('123abc def', 3)", "123abc Def")] + [InlineData("STRCONV('a' & CHRW(160) & 'b', 3)", "A b")] + [InlineData("STRCONV('a' & CHR(0) & 'b c' & CHR(13) & 'd' & CHR(10) & 'e' & CHR(11) & 'f' & CHR(12) & 'g.h,i(j' & CHR(9) & 'k-l/m', 3)", + "A\0B C\rD\nE\vF\fG.h,i(j\tK-l/m")] + [InlineData("STRCONV('abc', 3, 1055)", "Abc")] + [InlineData("STRCONV('iii', 1, 1055)", "III")] + [InlineData("STRCONV('abc', 1, 0)", "ABC")] + [InlineData("STRCONV(TRUE, 64)", "-\01\0")] + [InlineData("STRCONV(TRUE, 128)", "ㄭ")] + [InlineData("STRCONV(B, 128)", "䉁")] + [InlineData("STRCONV(STRCONV('abcd', 128), 64)", "abcd")] + public void Strconv_converts_in_the_ansi_code_page(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("ASCW(STRCONV('€a', 128))", 24960)] + [InlineData("ASCW(MID(STRCONV(CHRW(8364), 64), 1, 1))", 172)] + [InlineData("ASCW(MID(STRCONV(CHRW(8364), 64), 2, 1))", 32)] + [InlineData("LEN(STRCONV('abc', 128))", 1)] + public void Strconv_unicode_conversions_work_on_bytes(string expression, int expected) => + Assert.Equal(expected, Convert.ToInt32(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("STRCONV('abc', 4)")] + [InlineData("STRCONV('abc', 5)")] + [InlineData("STRCONV('abc', 65)")] + [InlineData("STRCONV('abc', 256)")] + [InlineData("STRCONV('abc', -1)")] + [InlineData("STRCONV(NULL, 256)")] + [InlineData("STRCONV(NULL, -1)")] + [InlineData("STRCONV('abc', 1, 20000)")] + [InlineData("STRCONV('abc', 1, -1)")] + public void Unsupported_conversions_are_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("STR('abc')")] + [InlineData("HEX('abc')")] + [InlineData("STR(G)")] + public void Text_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("VAL('1e400')")] + [InlineData("HEX(1E+20)")] + public void Values_past_their_type_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("STR(NULL)")] + [InlineData("VAL(NULL)")] + [InlineData("VAL(NT)")] + [InlineData("HEX(NULL)")] + [InlineData("STRCONV(NULL, 1)")] + [InlineData("STRCONV(NULL, 4)")] + [InlineData("STRCONV(NULL, 65)")] + [InlineData("STRCONV('abc', NULL)")] + [InlineData("STRCONV('abc', 1, NULL)")] + public void A_null_argument_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + // ACE keeps an odd trailing byte inside an expression; LibRed text has none, so it is dropped. + [Fact] + public void Strconv_from_unicode_drops_an_odd_byte() => + Assert.Equal("ab", Scalar("STRCONV(STRCONV('abc', 128), 64)")); +} diff --git a/test/LibRed.Engine.Tests/NumericFunctionTests.cs b/test/LibRed.Engine.Tests/NumericFunctionTests.cs new file mode 100644 index 000000000..9870ad7eb --- /dev/null +++ b/test/LibRed.Engine.Tests/NumericFunctionTests.cs @@ -0,0 +1,177 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The numeric functions Abs Int Fix Sgn Round Sqr Exp Log +/// Sin Cos Tan Atn Rnd: how each reads its argument, its domain and its rounding. +/// The expected values were measured against ACE, except that a Null argument gives Null where some of these +/// functions raise an error in ACE, and that Sin and Cos can differ from ACE's in the last binary digit. +/// +public class NumericFunctionTests(NumericFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, NT TEXT(60), DT DATETIME, G GUID)", + "INSERT INTO T (Id, DT) VALUES (1, #2020-01-02 12:00:00#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + ]; + + // Every Rnd case here reseeds with a negative argument, so the generator state the shared database carries from + // one test to the next never shows. + public sealed class Database() : SharedDatabase("numeric-functions-", Setup); + + // Text is read in the regional format, so each query runs under en-US whatever the machine's culture. + private object? Scalar(string expression) => database.Scalar($"SELECT {expression} FROM T", EnUs); + + private double Number(string expression) => + Convert.ToDouble(Scalar(expression), CultureInfo.InvariantCulture); + + [Theory] + [InlineData("INT(99.8)", 99)] + [InlineData("FIX(99.2)", 99)] + [InlineData("INT(-99.8)", -100)] + [InlineData("FIX(-99.8)", -99)] + [InlineData("INT(-8.4)", -9)] + [InlineData("FIX(-8.4)", -8)] + [InlineData("INT(TRUE)", -1)] + [InlineData("FIX(TRUE)", -1)] + [InlineData("INT('$5')", 5)] + [InlineData("FIX('1e2')", 100)] + [InlineData("ABS('-2.5')", 2.5)] + [InlineData("ABS('$5')", 5)] + [InlineData("ABS(DT)", 43832.5)] + [InlineData("ABS(#1899-12-29 06:00#)", 1.25)] + [InlineData("ABS(-2147483648)", 2147483648)] + [InlineData("ABS(CLNG(-2147483648))", 2147483648)] + [InlineData("SGN(TRUE)", -1)] + [InlineData("SGN(DT)", 1)] + [InlineData("SGN(#1899-12-29 06:00#)", -1)] + [InlineData("SGN('$5')", 1)] + [InlineData("SGN(-2.4)", -1)] + [InlineData("SGN(0)", 0)] + public void Abs_int_fix_and_sgn_read_their_argument_as_a_number(string expression, double expected) => + Assert.Equal(expected, Number(expression)); + + [Theory] + [InlineData("ABS('2.5')")] + [InlineData("INT('$5')")] + [InlineData("ABS(DT)")] + [InlineData("ABS(-2147483648)")] + public void Abs_int_and_fix_give_a_double_for_text_dates_and_a_long_past_its_range(string expression) => + Assert.IsType(Scalar(expression)); + + [Theory] + [InlineData("INT(DT)", "2020-01-02 00:00:00")] + [InlineData("FIX(DT)", "2020-01-02 00:00:00")] + [InlineData("INT(#1899-12-29 06:00#)", "1899-12-28 00:00:00")] + [InlineData("FIX(#1899-12-29 06:00#)", "1899-12-29 00:00:00")] + public void Int_and_fix_of_a_date_give_a_date(string expression, string expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)).ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("ROUND(2.5)", 2)] + [InlineData("ROUND(3.5)", 4)] + [InlineData("ROUND(-2.5)", -2)] + [InlineData("ROUND(0.12345, 4)", 0.1234)] + [InlineData("ROUND(0.12355, 4)", 0.1236)] + [InlineData("ROUND(0.12365, 4)", 0.1236)] + [InlineData("ROUND(2.675, 2)", 2.68)] + [InlineData("ROUND(CDBL(2.675), 2)", 2.68)] + [InlineData("ROUND(CDBL(0.12345), 4)", 0.1234)] + [InlineData("ROUND(CDBL(1.005), 2)", 1)] + [InlineData("ROUND('2.55', 1)", 2.6)] + [InlineData("ROUND(1.5, 1.6)", 1.5)] + [InlineData("ROUND(1.25, '1')", 1.2)] + [InlineData("ROUND(1234.5678, 20)", 1234.5678)] + [InlineData("ROUND(1.5, 400)", 1.5)] + [InlineData("ROUND(1.23456789012345678, 15)", 1.234567890123457)] + [InlineData("ROUND(1E20 / 3, 2)", 1E20 / 3)] + [InlineData("ROUND(1E300, 2)", 1E300)] + [InlineData("ROUND(TRUE)", -1)] + [InlineData("ROUND(DT)", 43832)] + [InlineData("ROUND(DT, 1)", 43832.5)] + [InlineData("ROUND(CCUR(1.23456), 2)", 1.23)] + [InlineData("ROUND(CINT(7), 1)", 7)] + public void Round_rounds_the_decimal_form_half_to_even(string expression, double expected) => + Assert.Equal(expected, Number(expression)); + + [Theory] + [InlineData("ROUND(NULL)")] + [InlineData("ROUND(2.5, NULL)")] + [InlineData("ROUND(NT)")] + [InlineData("ABS(NULL)")] + [InlineData("INT(NT)")] + [InlineData("SGN(NULL)")] + [InlineData("SQR(NULL)")] + [InlineData("LOG(NT)")] + [InlineData("SIN(NULL)")] + [InlineData("RND(NULL)")] + public void Null_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("SQR(DT)", 209.3621264699038)] + [InlineData("SQR('$5')", 2.23606797749979)] + [InlineData("EXP(TRUE)", 0.36787944117144233)] + [InlineData("EXP('$5')", 148.4131591025766)] + [InlineData("EXP(#1899-12-29 06:00#)", 0.2865047968601901)] + [InlineData("LOG(DT)", 10.688130830344283)] + [InlineData("LOG('$5')", 1.6094379124341003)] + [InlineData("SIN(TRUE)", -0.8414709848078965)] + [InlineData("SIN(DT)", 0.8410910067004128)] + [InlineData("COS(DT)", 0.5408936295129442)] + [InlineData("TAN(DT)", 1.5550026119882128)] + [InlineData("TAN(1.5)", 14.10141994717172)] + [InlineData("TAN(8)", -6.799711455220378)] + [InlineData("TAN(3000000000)", -6.142282632313941)] + [InlineData("TAN(40000)", 2.93421082407435)] + [InlineData("ATN(TRUE)", -0.7853981633974483)] + [InlineData("ATN(DT)", 1.5707735126729592)] + [InlineData("SIN(9.2233719999E18)", 0.9588847752004629)] + public void Math_functions_read_their_argument_as_a_number(string expression, double expected) => + Assert.Equal(expected, Number(expression)); + + [Theory] + [InlineData("SQR(-1)")] + [InlineData("SQR(TRUE)")] + [InlineData("SQR(CCUR(-3.25))")] + [InlineData("LOG(0)")] + [InlineData("LOG(FALSE)")] + [InlineData("LOG(TRUE)")] + [InlineData("LOG(-0.5)")] + [InlineData("SIN(9.223372E18)")] + [InlineData("COS(-1E300)")] + [InlineData("TAN(1E19)")] + [InlineData("ROUND(15, -1)")] + public void A_number_outside_the_domain_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("EXP(710)")] + [InlineData("EXP(40000)")] + [InlineData("RND(-1E300)")] + public void A_result_past_a_double_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("ABS('abc')")] + [InlineData("SQR(G)")] + [InlineData("ROUND('abc')")] + [InlineData("ROUND(2.5, 'x')")] + [InlineData("RND('abc')")] + public void A_value_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("RND(-1)", 0.2240070104598999)] + [InlineData("RND(TRUE)", 0.2240070104598999)] + [InlineData("RND('-1')", 0.2240070104598999)] + public void Rnd_with_a_negative_argument_reseeds(string expression, double expected) => + Assert.Equal(expected, Number(expression)); +} diff --git a/test/LibRed.Engine.Tests/OrderByPositionTests.cs b/test/LibRed.Engine.Tests/OrderByPositionTests.cs new file mode 100644 index 000000000..53b2cdcb3 --- /dev/null +++ b/test/LibRed.Engine.Tests/OrderByPositionTests.cs @@ -0,0 +1,64 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// ORDER BY n, where n is a whole number written as such, sorts by the nth output column, as ACE and SQL-92 +/// read it; any other constant sorts nothing. A position that names no column is an error. The expected orders +/// were measured against ACE. +/// +public class OrderByPositionTests(OrderByPositionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, X LONG, Y TEXT(5))", + "INSERT INTO T (Id, X, Y) VALUES (1, 30, 'b')", + "INSERT INTO T (Id, X, Y) VALUES (2, 10, 'c')", + "INSERT INTO T (Id, X, Y) VALUES (3, 20, 'a')", + ]; + + public sealed class Database() : SharedDatabase("order-position-", Setup); + + private string Column(string sql, int column = 0) => string.Join("|", + database.Query(sql, CultureInfo.InvariantCulture).Rows.Select(row => row[column])); + + [Theory] + [InlineData("SELECT Id FROM T ORDER BY 1 DESC", "3|2|1")] + [InlineData("SELECT Id, X FROM T ORDER BY 2", "2|3|1")] + [InlineData("SELECT Id, X, Y FROM T ORDER BY 3", "3|1|2")] + [InlineData("SELECT Id, X FROM T ORDER BY 2 DESC, 1", "1|3|2")] + [InlineData("SELECT Id, X FROM T ORDER BY (2)", "2|3|1")] + [InlineData("SELECT Id, X AS Z FROM T ORDER BY 2", "2|3|1")] + [InlineData("SELECT Id, X FROM T ORDER BY 2, X DESC", "2|3|1")] + [InlineData("SELECT TOP 1 Id, X FROM T ORDER BY 2 DESC", "1")] + [InlineData("SELECT Id FROM T UNION ALL SELECT 0 FROM T WHERE Id = 1 ORDER BY 1", "0|1|2|3")] + [InlineData("SELECT X, COUNT(*) FROM T GROUP BY X ORDER BY 1 DESC", "30|20|10")] + [InlineData("SELECT Id FROM (SELECT Id, X FROM T ORDER BY 2) AS D", "2|3|1")] + public void A_whole_number_names_an_output_column(string sql, string expected) => + Assert.Equal(expected, Column(sql)); + + [Fact] + public void A_star_is_counted_through() + { + Assert.Equal("c|a|b", Column("SELECT * FROM T ORDER BY 2", column: 2)); + Assert.Equal("3|1|2", Column("SELECT * FROM T ORDER BY 3")); + } + + [Theory] + [InlineData("SELECT Id, X FROM T ORDER BY 1.5")] + [InlineData("SELECT Id, X FROM T ORDER BY 1 + 1")] + [InlineData("SELECT Id, X FROM T ORDER BY '2'")] + public void Any_other_constant_sorts_nothing(string sql) => + Assert.Equal("1|2|3", Column(sql)); + + [Theory] + [InlineData("SELECT Id FROM T ORDER BY 2")] + [InlineData("SELECT Id, X FROM T ORDER BY 0")] + [InlineData("SELECT Id, X FROM T ORDER BY -1")] + [InlineData("SELECT * FROM T ORDER BY 4")] + [InlineData("SELECT Id FROM T UNION ALL SELECT 0 FROM T ORDER BY 2")] + public void A_position_that_names_no_column_is_an_error(string sql) => + Assert.Throws(() => Column(sql)); +} diff --git a/test/LibRed.Engine.Tests/ParameterDeclaredTypeTests.cs b/test/LibRed.Engine.Tests/ParameterDeclaredTypeTests.cs new file mode 100644 index 000000000..2c7e19780 --- /dev/null +++ b/test/LibRed.Engine.Tests/ParameterDeclaredTypeTests.cs @@ -0,0 +1,66 @@ +using LibRed; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +// A parameter is typed by the value bound to it, as a literal is, so an expression over one declares its result type +// instead of leaving the reader to guess from the first row — which a Null there turns into Object. +public class ParameterDeclaredTypeTests : TempDatabaseTest +{ + private static QueryEngine Fresh() + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "param-type-"); + var engine = new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); + engine.ExecuteNonQuery("CREATE TABLE Q (K LONG, A DATETIME, I LONG)"); + // The first row's date is Null: the declared type cannot come from it. + engine.ExecuteNonQuery("INSERT INTO Q (K, A, I) VALUES (1, NULL, 1)"); + engine.ExecuteNonQuery("INSERT INTO Q (K, A, I) VALUES (2, #2020-01-02 12:00:00#, 2)"); + return engine; + } + + private static readonly DateTime SixAm = new(1899, 12, 30, 6, 0, 0); + + [Theory] + [InlineData("A + @p", typeof(DateTime))] + [InlineData("@p + A", typeof(DateTime))] + [InlineData("A - @p", typeof(double))] // a date less a date is a day count + public void An_expression_over_a_date_parameter_declares_its_type(string expression, Type expected) + { + var result = Fresh().ExecuteQuery($"SELECT {expression} AS c FROM Q ORDER BY K", + new Dictionary { ["p"] = SixAm }); + + var rows = result.Rows.ToList(); + Assert.Equal(expected, result.ColumnTypes[0]); + Assert.Null(rows[0][0]); + Assert.IsType(expected, rows[1][0]); + } + + [Theory] + [InlineData(5, typeof(int))] + [InlineData(5L, typeof(long))] + [InlineData(2.5, typeof(double))] + public void A_number_parameter_types_the_arithmetic_it_is_in(object value, Type expected) + { + var result = Fresh().ExecuteQuery("SELECT I + @p AS c FROM Q ORDER BY K", + new Dictionary { ["p"] = value }); + + Assert.Equal(expected, result.ColumnTypes[0]); + Assert.All(result.Rows, row => Assert.IsType(expected, row[0])); + } + + [Fact] + public void A_parameter_on_its_own_declares_its_value_type() + { + var result = Fresh().ExecuteQuery("SELECT @p AS c FROM Q", + new Dictionary { ["p"] = SixAm }); + Assert.Equal(typeof(DateTime), result.ColumnTypes[0]); + Assert.All(result.Rows, row => Assert.Equal(SixAm, row[0])); + } + + [Fact] + public void A_null_parameter_declares_nothing() => + Assert.Equal(typeof(object), Fresh().ExecuteQuery("SELECT @p AS c FROM Q", + new Dictionary { ["p"] = null }).ColumnTypes[0]); +} diff --git a/test/LibRed.Engine.Tests/PercentileTests.cs b/test/LibRed.Engine.Tests/PercentileTests.cs new file mode 100644 index 000000000..4d41b3ab4 --- /dev/null +++ b/test/LibRed.Engine.Tests/PercentileTests.cs @@ -0,0 +1,124 @@ +using System.Globalization; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The standard's inverse distribution functions, PERCENTILE_CONT and PERCENTILE_DISC … WITHIN GROUP (ORDER +/// BY …), grouped and over a window. Access has neither; this is a LibRed extension. +/// +public class PercentileTests(PercentileTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Group 'a' has V 1-4, dates 2 days apart then 6, and text in reverse order; group 'b' has a Null V. S holds 1-10. + private static readonly string[] Setup = + [ + "CREATE TABLE P (Id LONG, G TEXT(10), V LONG, D DATETIME, T TEXT(10))", + "INSERT INTO P (Id, G, V, D, T) VALUES (1, 'a', 1, #2020-01-01#, 'd')", + "INSERT INTO P (Id, G, V, D, T) VALUES (2, 'a', 2, #2020-01-03#, 'c')", + "INSERT INTO P (Id, G, V, D, T) VALUES (3, 'a', 3, #2020-01-05#, 'b')", + "INSERT INTO P (Id, G, V, D, T) VALUES (4, 'a', 4, #2020-01-11#, 'a')", + "INSERT INTO P (Id, G, V, D, T) VALUES (5, 'b', 10, NULL, NULL)", + "INSERT INTO P (Id, G, V, D, T) VALUES (6, 'b', 20, #2020-02-01#, 'x')", + "INSERT INTO P (Id, G, V, D, T) VALUES (7, 'b', NULL, #2020-02-02#, 'y')", + "CREATE TABLE S (N LONG)", + .. Enumerable.Range(1, 10).Select(n => $"INSERT INTO S (N) VALUES ({n})"), + ]; + + public sealed class Database() : SharedDatabase("percentile-", Setup); + + private (IReadOnlyList Types, List Rows) Query(string sql) => + database.Query(sql, CultureInfo.InvariantCulture); + + private string ByGroup(string aggregate) => string.Join(" ", + Query($"SELECT G, {aggregate} AS r FROM P GROUP BY G ORDER BY G").Rows + .Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V)", "a:2.5 b:15")] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY V)", "a:2 b:10")] + [InlineData("PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY V)", "a:1.75 b:12.5")] + [InlineData("PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY V)", "a:1 b:10")] + [InlineData("PERCENTILE_CONT(0) WITHIN GROUP (ORDER BY V)", "a:1 b:10")] + [InlineData("PERCENTILE_CONT(1) WITHIN GROUP (ORDER BY V)", "a:4 b:20")] + [InlineData("PERCENTILE_DISC(0) WITHIN GROUP (ORDER BY V)", "a:1 b:10")] + [InlineData("PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY V DESC)", "a:3.25 b:17.5")] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY V DESC)", "a:3 b:20")] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY T)", "a:b b:x")] + [InlineData("PERCENTILE_CONT(NULL) WITHIN GROUP (ORDER BY V)", "a: b:")] + public void A_percentile_reads_the_ordered_values_of_the_group(string aggregate, string expected) => + Assert.Equal(expected, ByGroup(aggregate)); + + [Fact] + public void Nulls_take_no_part_and_no_values_give_null() + { + Assert.Equal(3.5, database.Scalar("SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V) FROM P", CultureInfo.InvariantCulture)); + Assert.Null(database.Scalar("SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V) FROM P WHERE Id > 100", CultureInfo.InvariantCulture)); + } + + [Fact] + public void The_fraction_is_read_as_written() + { + // 0.7 × 10 is 7 exactly, where the Double product is 7.000000000000001 and would take the 8th value. + Assert.Equal(7, database.Scalar("SELECT PERCENTILE_DISC(0.7) WITHIN GROUP (ORDER BY N) FROM S", CultureInfo.InvariantCulture)); + Assert.Equal(7.3, (double)database.Scalar("SELECT PERCENTILE_CONT(0.7) WITHIN GROUP (ORDER BY N) FROM S", CultureInfo.InvariantCulture)!, 12); + } + + [Fact] + public void Dates_interpolate_along_the_timeline() => + Assert.Equal(new DateTime(2020, 1, 4), Query( + "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY D) AS r FROM P WHERE G = 'a'").Rows[0][0]); + + [Fact] + public void Having_can_test_a_percentile() => + Assert.Equal("b", database.Scalar( + "SELECT G FROM P GROUP BY G HAVING PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V) > 10", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V)", typeof(double))] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY V)", typeof(int))] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY T)", typeof(string))] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY D)", typeof(DateTime))] + public void The_column_type_follows_the_key(string aggregate, Type expected) + { + var (groupedTypes, grouped) = Query($"SELECT {aggregate} AS r FROM P"); + var (windowedTypes, windowed) = Query($"SELECT {aggregate} OVER () AS r FROM P"); + + Assert.Equal(expected, groupedTypes[0]); + Assert.Equal(expected, windowedTypes[0]); + Assert.IsType(expected, grouped[0][0]); + Assert.All(windowed, row => Assert.Equal(grouped[0][0], row[0])); + } + + private string ById(string expression) => string.Join(" ", + Query($"SELECT Id, {expression} AS r FROM P ORDER BY Id").Rows + .Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V) OVER (PARTITION BY G)", "1:2.5 2:2.5 3:2.5 4:2.5 5:15 6:15 7:15")] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V) OVER (ORDER BY Id)", "1:1 2:1.5 3:2 4:2.5 5:3 6:3.5 7:3.5")] + [InlineData("PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY V DESC) OVER (ORDER BY Id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", + "1:2 2:2 3:3 4:4 5:10 6:20 7:20")] + public void Over_a_window_a_percentile_reads_each_frame(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("PERCENTILE_CONT(1.5) WITHIN GROUP (ORDER BY V)")] + [InlineData("PERCENTILE_DISC(-0.1) WITHIN GROUP (ORDER BY V)")] + public void A_fraction_outside_zero_to_one_is_an_invalid_procedure_call(string aggregate) => + Assert.Throws(() => ByGroup(aggregate)); + + [Fact] + public void Text_cannot_be_interpolated() => + Assert.Throws(() => ByGroup("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY T)")); + + [Theory] + [InlineData("PERCENTILE_CONT(0.5)")] + [InlineData("SUM(V) WITHIN GROUP (ORDER BY V)")] + [InlineData("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY V, Id)")] + [InlineData("PERCENTILE_CONT(0.5, 1) WITHIN GROUP (ORDER BY V)")] + [InlineData("PERCENTILE_CONT(DISTINCT 0.5) WITHIN GROUP (ORDER BY V)")] + public void The_within_group_syntax_belongs_to_the_percentiles(string aggregate) => + Assert.Throws(() => ByGroup(aggregate)); +} diff --git a/test/LibRed.Engine.Tests/PlusAndConcatenationTests.cs b/test/LibRed.Engine.Tests/PlusAndConcatenationTests.cs new file mode 100644 index 000000000..a4f698f35 --- /dev/null +++ b/test/LibRed.Engine.Tests/PlusAndConcatenationTests.cs @@ -0,0 +1,227 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The + and & operators as ACE evaluates them. + concatenates only two texts and +/// otherwise adds, reading text as a number; & writes each operand as text and binds looser than +/// +. The expected values were measured against ACE. +/// +public class PlusAndConcatenationTests(PlusAndConcatenationTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private const string Guid = "{00112233-4455-6677-8899-AABBCCDDEEFF}"; + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, N LONG, S TEXT(60), NT TEXT(60), D DATETIME, G GUID, B BINARY(4), M DECIMAL(18,4))", + "INSERT INTO T (Id, N, S, D, M) VALUES (1, 3, 'a', #2020-01-02 12:00:00#, 4.5)", + $"UPDATE T SET G = {Guid}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("plus-concat-", Setup); + + private static QueryEngine Fresh() => SharedDatabase.Fresh("plus-concat-", Setup); + + private object? Scalar(string expression) => Scalar(database.Engine, expression); + + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + // Both operators follow the regional settings (date format, separators, currency symbol), so each + // expression is evaluated under en-US whatever the machine's culture. + private static object? Scalar(QueryEngine engine, string expression) => + SharedDatabase.Scalar(engine, $"SELECT {expression} FROM T", EnUs); + + [Theory] + [InlineData("'1' + 1", 2d)] + [InlineData("1 + '1'", 2d)] + [InlineData("N + '2.5'", 5.5d)] + [InlineData("' 7 ' + 0", 7d)] + [InlineData("' - 5' + 0", -5d)] + [InlineData("'+5' + 0", 5d)] + [InlineData("'5-' + 0", -5d)] + [InlineData("'5 -' + 0", -5d)] + [InlineData("'(5)' + 0", -5d)] + [InlineData("'( 5 )' + 0", -5d)] + [InlineData("'($5)' + 0", -5d)] + [InlineData("'$(5)' + 0", -5d)] + [InlineData("'-$5' + 0", -5d)] + [InlineData("'$-5' + 0", -5d)] + [InlineData("'5$' + 0", 5d)] + [InlineData("'$ 5' + 0", 5d)] + [InlineData("'.5' + 0", 0.5d)] + [InlineData("'1.' + 0", 1d)] + [InlineData("'1,000.5' + 0", 1000.5d)] + [InlineData("'1,5' + 0", 15d)] + [InlineData("'1,,000' + 0", 1000d)] + [InlineData("'1.000,5' + 0", 1.0005d)] + [InlineData("'1e2' + 0", 100d)] + [InlineData("'1E-2' + 0", 0.01d)] + [InlineData("'1d2' + 0", 100d)] + [InlineData("'1D-2' + 0", 0.01d)] + [InlineData("'1.e2' + 0", 100d)] + [InlineData("'1e-400' + 0", 0d)] + [InlineData("'-0' + 0", 0d)] + [InlineData("'1234567890123456789' + 0", 1234567890123456789d)] + [InlineData("'&H10' + 0", 16d)] + [InlineData("'&h10' + 0", 16d)] + [InlineData("'&HFFFF' + 0", 65535d)] + [InlineData("'&H7FFFFFFF' + 0", 2147483647d)] + [InlineData("'&H80000000' + 0", -2147483648d)] + [InlineData("'&HFFFFFFFF' + 0", -1d)] + [InlineData("'&H100000000' + 0", 4294967296d)] + [InlineData("'&O17' + 0", 15d)] + [InlineData("'&O177777' + 0", 65535d)] + [InlineData("TRUE + '1'", 0d)] + [InlineData("'1' + '1' + 1", 12d)] + [InlineData("'1' + 1 + '1'", 3d)] + [InlineData("'1' + ('1' + 1)", 3d)] + public void Plus_reads_text_with_anything_but_text_as_a_number(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("CHR(9)")] + [InlineData("CHR(10)")] + [InlineData("CHR(160)")] + [InlineData("CHRW(12288)")] + public void Plus_skips_whitespace_around_a_text_number(string whitespace) + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery($"UPDATE T SET S = {whitespace} & '5' & {whitespace}"); + Assert.Equal(6d, Scalar(engine, "S + 1")); + } + + [Theory] + [InlineData("'abc' + 1")] + [InlineData("1 + 'abc'")] + [InlineData("'' + 1")] + [InlineData("' ' + 1")] + [InlineData("S + N")] + [InlineData("UCASE('abc') + 1")] + [InlineData("'1 2' + 0")] + [InlineData("'3 .1 4' + 0")] + [InlineData("'--5' + 0")] + [InlineData("'+5-' + 0")] + [InlineData("'(-5)' + 0")] + [InlineData("'-(5)' + 0")] + [InlineData("'-' + 0")] + [InlineData("'$' + 0")] + [InlineData("'()' + 0")] + [InlineData("'.' + 0")] + [InlineData("',' + 0")] + [InlineData("',5' + 0")] + [InlineData("'1.5.2' + 0")] + [InlineData("'1e' + 0")] + [InlineData("'1e 2' + 0")] + [InlineData("'.e2' + 0")] + [InlineData("'1E2.5' + 0")] + [InlineData("'&H' + 0")] + [InlineData("'&HG' + 0")] + [InlineData("'-&H10' + 0")] + [InlineData("'0x10' + 0")] + [InlineData("'12abc' + 0")] + [InlineData("'5%' + 0")] + [InlineData("'True' + 0")] + [InlineData("'#5' + 0")] + [InlineData("'2020-01-01' + 0")] + [InlineData("'12:00' + 0")] + [InlineData("'５' + 0")] + [InlineData("'abc' + D")] + public void Plus_with_text_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("1 + G")] + [InlineData("G + 1")] + [InlineData("B + N")] + [InlineData("D + G")] + [InlineData("TRUE + B")] + public void Plus_with_a_guid_or_binary_value_and_a_non_text_value_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("'1e400' + 0")] + [InlineData("2147483647 + 1")] + [InlineData("N + 2147483647")] + public void Plus_past_the_result_type_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("'1' + D", "2020-01-03T12:00:00")] + [InlineData("D + '2.5'", "2020-01-05T00:00:00")] + [InlineData("' 1 ' + #2020-01-02#", "2020-01-03T00:00:00")] + public void Plus_with_text_and_a_date_is_a_date(string expression, string expected) => + Assert.Equal(DateTime.Parse(expected, CultureInfo.InvariantCulture), Assert.IsType(Scalar(expression))); + + [Theory] + [InlineData("'1' + '1'", "11")] + [InlineData("S + '1'", "a1")] + [InlineData("'1' + B", "1AB")] + [InlineData("'abc' + G", "abc" + Guid)] + [InlineData("G + 'abc'", Guid + "abc")] + [InlineData("G + G", Guid + Guid)] + [InlineData("B + G", "AB" + Guid)] + public void Plus_concatenates_two_texts(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'1' + NT")] + [InlineData("NT + 1")] + [InlineData("NULL + 'a'")] + public void Plus_propagates_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("1 & NT", "1")] + [InlineData("NULL & 'a'", "a")] + [InlineData("'' & TRUE", "-1")] + [InlineData("FALSE & ''", "0")] + [InlineData("'' & N", "3")] + [InlineData("'' & M", "4.5")] + [InlineData("'' & CCUR(1.5)", "1.5")] + [InlineData("'' & G", Guid)] + [InlineData("'' & B", "AB")] + [InlineData("'' & 1/3", "0.333333333333333")] + [InlineData("0.1 + 0.2 & ''", "0.3")] + [InlineData("'' & -0.5", "-0.5")] + [InlineData("'' & 1E300", "1E+300")] + [InlineData("'' & -1E-300", "-1E-300")] + [InlineData("'' & CSNG(1/3)", "0.3333333")] + [InlineData("'' & CSNG(10000000)", "1E+07")] + [InlineData("'' & CSNG(1E-5)", "0.00001")] + [InlineData("'' & #2020-01-02#", "1/2/2020")] + [InlineData("'' & #0100-01-01#", "1/1/100")] + public void Ampersand_writes_each_operand_as_text(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Fact] + public void Ampersand_writes_a_time_alone_on_the_epoch_day_and_both_otherwise() + { + // A plain space before AM/PM on every platform, as ACE writes it, though newer ICU data has a narrow one. + Assert.Equal("1:00:00 PM", Scalar("'' & #1899-12-30 13:00:00#")); + Assert.Equal("1/2/2020 12:00:00 PM", Scalar("D & ''")); + } + + [Theory] + [InlineData("NT & NT")] + [InlineData("NULL & NULL")] + public void Ampersand_is_null_when_both_sides_are(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("1 & 2 + 3", "15")] + [InlineData("'a' & 1 + 2", "a3")] + [InlineData("1 + 2 & 3", "33")] + [InlineData("1 & 2 & 3", "123")] + public void Ampersand_binds_looser_than_plus(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("(1 & 2) + 3", 15d)] + [InlineData("1 + (2 & 3)", 24d)] + public void A_bracketed_ampersand_result_adds_as_a_number(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression))); +} diff --git a/test/LibRed.Engine.Tests/PredicateOperatorTests.cs b/test/LibRed.Engine.Tests/PredicateOperatorTests.cs new file mode 100644 index 000000000..4ee724787 --- /dev/null +++ b/test/LibRed.Engine.Tests/PredicateOperatorTests.cs @@ -0,0 +1,335 @@ +using LibRed.Engine; +using LibRed.Engine.Plan; +using LibRed.Sql.Ast; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The predicates LIKE, BETWEEN, IN and IS NULL. LIKE takes the ANSI-92 wildcards EF +/// emits, BETWEEN takes its bounds in either order, and IN is unknown when it misses and meets a Null item. The +/// expected values were measured against ACE, except for that last rule, where ACE skips the Null item. +/// +public class PredicateOperatorTests(PredicateOperatorTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, TN TEXT(60), NT TEXT(60), D DATETIME, G GUID, B BINARY(4), SG REAL, DC DECIMAL(18,4))", + "INSERT INTO T (Id, TN, D, SG, DC) VALUES (1, '7', #2020-01-02 12:00:00#, 1.5, 4.5)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("predicate-ops-", Setup); + + private static QueryEngine Fresh() => SharedDatabase.Fresh("predicate-ops-", Setup); + + private object? Query(string sql) => database.Engine.ExecuteQuery(sql).Rows.First()[0]; + + private object? Scalar(string expression) => Query($"SELECT {expression} FROM T"); + + private object? Count(string condition) => Query($"SELECT COUNT(*) FROM T WHERE {condition}"); + + [Theory] + [InlineData("'abc' LIKE '%'", true)] + [InlineData("'' LIKE '%'", true)] + [InlineData("'abc' LIKE '%%%'", true)] + [InlineData("'abc' LIKE 'A%'", true)] + [InlineData("'abc' LIKE '_b_'", true)] + [InlineData("'ab' LIKE '_%'", true)] + [InlineData("'abc' LIKE '%b%'", true)] + [InlineData("'aBBBa' LIKE 'a%a'", true)] + [InlineData("'abc' LIKE 'a%a'", false)] + [InlineData("'abc' LIKE ''", false)] + [InlineData("'' LIKE ''", true)] + [InlineData("'x_y' LIKE 'x[_]y'", true)] + [InlineData("'x%y' LIKE 'x[%]y'", true)] + [InlineData("'_' LIKE '%[_]'", true)] + public void Percent_and_underscore_are_the_wildcards(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'abc' LIKE '*'", false)] + [InlineData("'abc' LIKE 'a*'", false)] + [InlineData("'a*a' LIKE 'a*a'", true)] + [InlineData("'aBBBa' LIKE 'a*a'", false)] + [InlineData("'a1a' LIKE 'a?a'", false)] + [InlineData("'a?a' LIKE 'a?a'", true)] + [InlineData("'a1a' LIKE 'a#a'", false)] + [InlineData("'ab' LIKE '??'", false)] + public void Star_question_mark_and_hash_are_plain_characters(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'abc' LIKE '[a-z]%'", true)] + [InlineData("'E' LIKE '[a-e]'", true)] + [InlineData("'é' LIKE '[a-e]'", false)] + [InlineData("'abc' LIKE '[!b]%'", true)] + [InlineData("'b' LIKE '[!b]%'", false)] + [InlineData("'a[b' LIKE 'a[[]b'", true)] + [InlineData("'a]b' LIKE 'a]b'", true)] + [InlineData("'a]b' LIKE 'a[]]b'", true)] + [InlineData("'a-b' LIKE 'a[-]b'", true)] + [InlineData("'-' LIKE '[a-]'", true)] + [InlineData("'-' LIKE '[-a]'", true)] + [InlineData("'b' LIKE '[a-c-e]'", true)] + [InlineData("'-' LIKE '[a-c-e]'", true)] + [InlineData("'a!b' LIKE 'a[!]b'", true)] + [InlineData("'!' LIKE '[!!]'", false)] + [InlineData("'z' LIKE '[!!]'", true)] + [InlineData("'!' LIKE '[!-a]'", true)] + [InlineData("'-' LIKE '[!-a]'", false)] + [InlineData("'abc' LIKE '[]abc'", true)] + [InlineData("'' LIKE '[]'", true)] + [InlineData("'b' LIKE '%[]'", true)] + [InlineData("'[]' LIKE '[[]]'", true)] + [InlineData("'bill' LIKE 'b[!ae]ll'", true)] + [InlineData("'bill' LIKE 'b[^ae]ll'", false)] + [InlineData("'x' LIKE '[^b]%'", false)] + [InlineData("'b' LIKE '[^b]%'", true)] + [InlineData("'a^b' LIKE 'a[^]b'", true)] + [InlineData("'^' LIKE '[!^]'", false)] + [InlineData("'2' LIKE '[^a-z]'", false)] + public void Brackets_list_one_character(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'abc' LIKE 'ABC'", true)] + [InlineData("'E' LIKE 'e'", true)] + [InlineData("'é' LIKE 'e'", false)] + [InlineData("'café' LIKE 'cafe'", false)] + [InlineData("'É' LIKE '[É]'", true)] + [InlineData("'é' LIKE '[É]'", true)] + [InlineData("'aßb' LIKE 'ass%'", true)] + [InlineData("'aSSb' LIKE 'a[ß]b'", true)] + [InlineData("'aßb' LIKE 'a[s]sb'", true)] + [InlineData("'ss' LIKE '[ß]'", true)] + [InlineData("'Æ' LIKE 'ae'", true)] + [InlineData("'Æ' LIKE '[æ]'", true)] + [InlineData("'Æ' LIKE '[a-z]%'", true)] + [InlineData("'Æ' LIKE '[a-z]'", false)] + [InlineData("'ß' LIKE '_'", true)] + [InlineData("'ß' LIKE '__'", false)] + [InlineData("'aßb' LIKE 'a_b'", true)] + [InlineData("'ß' LIKE 's_'", true)] + [InlineData("'ß' LIKE '_s'", false)] + [InlineData("'fi' LIKE 'fi'", false)] + public void Case_is_ignored_accents_are_not_and_sharp_s_and_ae_expand(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("TRUE LIKE '-1'", true)] + [InlineData("123 LIKE 123", true)] + [InlineData("TN LIKE 7", true)] + [InlineData("1 + 1 LIKE '2'", true)] + [InlineData("12.5 LIKE '12.5'", true)] + [InlineData("'abc' LIKE 1", false)] + public void Other_values_are_matched_as_their_text(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NULL LIKE '%'", false)] + [InlineData("NT LIKE '%'", false)] + [InlineData("NULL NOT LIKE '%'", true)] + [InlineData("NOT NULL LIKE '%'", true)] + public void A_pattern_of_just_percent_is_false_for_null(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NULL LIKE '%%'")] + [InlineData("NULL LIKE 'a%'")] + [InlineData("NULL LIKE '%' & ''")] + [InlineData("NULL LIKE '['")] + [InlineData("'abc' LIKE NULL")] + [InlineData("NT LIKE NT")] + [InlineData("NULL LIKE 1")] + public void Otherwise_null_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("'abc' LIKE 'a[b'")] + [InlineData("'z' LIKE '['")] + [InlineData("'z' LIKE '[!'")] + [InlineData("'abc' LIKE '[z-a]'")] + [InlineData("'b' LIKE '%[z-a]'")] + public void An_unclosed_bracket_or_backwards_range_is_an_invalid_pattern(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Fact] + public void There_is_no_escape_clause() => + Assert.Throws(() => Scalar(@"'a%b' LIKE 'a\%b' ESCAPE '\'")); + + [Theory] + [InlineData("'' LIKE '['")] + [InlineData("' ' LIKE 'a[b'")] + [InlineData("'' LIKE '[z-a]'")] + [InlineData("'' LIKE '%[z-a]'")] + [InlineData("'z' LIKE 'x[z-a]'")] + public void An_invalid_pattern_is_only_reported_when_the_match_reaches_it(string expression) => + Assert.Equal(false, Scalar(expression)); + + [Theory] + [InlineData("TN LIKE '%'", 1)] + [InlineData("NT LIKE '%'", 0)] + [InlineData("NT NOT LIKE '%'", 1)] + [InlineData("'abc' LIKE 'A%'", 1)] + [InlineData("'abc' LIKE 'A*'", 0)] + public void A_where_condition_matches_the_same_way(string condition, int expected) => + Assert.Equal(expected, Count(condition)); + + [Theory] + [InlineData("5 BETWEEN 1 AND 10", true)] + [InlineData("5 BETWEEN 10 AND 1", true)] + [InlineData("1 BETWEEN 1 AND 1", true)] + [InlineData("10 BETWEEN 1 AND 10", true)] + [InlineData("15 BETWEEN 1 AND 10", false)] + [InlineData("5 NOT BETWEEN 1 AND 10", false)] + [InlineData("5 NOT BETWEEN 10 AND 1", false)] + [InlineData("NOT 5 BETWEEN 1 AND 10", false)] + [InlineData("'b' BETWEEN 'c' AND 'a'", true)] + [InlineData("'B' BETWEEN 'a' AND 'c'", true)] + [InlineData("'2' BETWEEN '1' AND '3'", true)] + [InlineData("D BETWEEN #2020-01-01# AND #2020-01-03#", true)] + [InlineData("D BETWEEN 43833 AND 43832", true)] + [InlineData("TRUE BETWEEN -1 AND 0", true)] + [InlineData("2 BETWEEN TRUE AND 3", true)] + [InlineData("SG BETWEEN 1.4 AND 1.6", true)] + [InlineData("B BETWEEN 'A' AND 'B'", true)] + [InlineData("G BETWEEN 'a' AND '{z'", false)] + [InlineData("LEFT('7', 1) BETWEEN 1 AND 10", true)] + [InlineData("5 BETWEEN 1 + 1 AND 2 * 5", true)] + public void Between_is_inclusive_with_the_bounds_in_either_order(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NULL BETWEEN 1 AND 2")] + [InlineData("5 BETWEEN NULL AND 10")] + [InlineData("5 BETWEEN 1 AND NULL")] + [InlineData("15 BETWEEN NULL AND 10")] + [InlineData("0 BETWEEN 1 AND NULL")] + [InlineData("15 NOT BETWEEN NULL AND 10")] + public void Between_is_null_when_any_operand_is(string expression) => + Assert.Null(Scalar(expression)); + + [Fact] + public void Between_reads_text_that_is_not_a_number_as_a_type_mismatch() => + Assert.Throws(() => Scalar("'abc' BETWEEN 1 AND 2")); + + [Theory] + [InlineData("5 BETWEEN 10 AND 1 AND TRUE", true)] + [InlineData("5 BETWEEN 10 AND 1 AND 0", false)] + [InlineData("5 BETWEEN 1 AND 10 AND 1 = 2", false)] + [InlineData("5 BETWEEN 1 AND 10 OR 1 = 2", true)] + [InlineData("5 BETWEEN 1 AND 10 AND 2 BETWEEN 3 AND 1", true)] + [InlineData("5 BETWEEN 1 AND 3 OR 4 AND 5 = 5", true)] + [InlineData("5 BETWEEN (1 AND 1) AND 10", true)] + [InlineData("5 BETWEEN 1 AND 10 XOR TRUE", false)] + [InlineData("NOT 5 BETWEEN 1 AND 10 AND TRUE", false)] + [InlineData("5 BETWEEN 1 + 1 AND 10 - 1 AND TRUE", true)] + [InlineData("2 BETWEEN 1 AND 3 = TRUE", false)] + public void The_and_after_the_lower_bound_belongs_to_between(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Fact] + public void An_indexed_between_seeks_from_the_lower_literal_bound() + { + QueryEngine engine = Fresh(); + engine.ExecuteNonQuery("CREATE TABLE K (Id LONG PRIMARY KEY, V LONG)"); + for (int i = 1; i <= 10; i++) + engine.ExecuteNonQuery($"INSERT INTO K (Id, V) VALUES ({i}, {i})"); + + const string sql = "SELECT COUNT(*) FROM K WHERE Id BETWEEN 8 AND 3"; + Assert.Equal(6, engine.ExecuteQuery(sql).Rows.Single()[0]); + IndexRangeSeekNode seek = FindSeek(engine.PlanFor(sql))!; + Assert.Equal(3, Convert.ToInt32(Assert.IsType(seek.Low).Value)); + Assert.Equal(8, Convert.ToInt32(Assert.IsType(seek.High).Value)); + + Assert.Equal(4, engine.ExecuteQuery("SELECT COUNT(*) FROM K WHERE Id NOT BETWEEN 8 AND 3").Rows.Single()[0]); + Assert.Equal(0, engine.ExecuteQuery("SELECT COUNT(*) FROM K WHERE Id BETWEEN NULL AND 3").Rows.Single()[0]); + Assert.Null(FindSeek(engine.PlanFor("SELECT COUNT(*) FROM K WHERE Id BETWEEN NULL AND 3"))); + Assert.Equal(1, engine.ExecuteQuery("SELECT COUNT(*) FROM K WHERE Id BETWEEN 1 AND 10 AND V = 2").Rows.Single()[0]); + Assert.Equal(6, engine.ExecuteQuery( + "SELECT COUNT(*) FROM K WHERE Id BETWEEN @a AND @b", + new Dictionary { ["a"] = 8, ["b"] = 3 }).Rows.Single()[0]); + + static IndexRangeSeekNode? FindSeek(PlanNode node) => + node as IndexRangeSeekNode ?? node.Children.Select(FindSeek).FirstOrDefault(s => s is not null); + } + + [Theory] + [InlineData("5 IN (1, 5)", true)] + [InlineData("5 IN (5, NULL)", true)] + [InlineData("5 NOT IN (5, NULL)", false)] + [InlineData("5 NOT IN (1, 2)", true)] + [InlineData("NOT 5 IN (5)", false)] + [InlineData("LEFT('7', 1) IN (7)", true)] + [InlineData("1 IN (TRUE)", false)] + [InlineData("-1 IN (TRUE)", true)] + [InlineData("5 IN (TRUE)", false)] + [InlineData("TRUE IN (5)", false)] + [InlineData("0 IN (FALSE)", true)] + [InlineData("'0' IN (FALSE)", true)] + [InlineData("'A' IN ('a')", true)] + [InlineData("'a ' IN ('a')", true)] + [InlineData("D IN (43832.5)", true)] + [InlineData("D IN (#2020-01-02 12:00#)", true)] + [InlineData("G IN ('{00112233-4455-6677-8899-AABBCCDDEEFF}')", true)] + [InlineData("B IN ('AB')", true)] + [InlineData("1 IN (1.0)", true)] + [InlineData("SG IN (1.5)", true)] + [InlineData("CSNG(1.1) IN (1.1)", true)] + [InlineData("4.5 IN (DC)", true)] + [InlineData("1.5 IN (DC)", false)] + [InlineData("5 IN (5, 'abc')", true)] + [InlineData("1 IN (1, 2) AND 1 = 2", false)] + public void In_compares_each_item_as_equals_does(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + // Standard SQL: a miss is unknown when an item is Null. ACE instead skips the Null item, making these False and + // True; LibRed follows the standard here. + [Theory] + [InlineData("NULL IN (1)")] + [InlineData("NULL IN (NULL)")] + [InlineData("NULL NOT IN (1)")] + [InlineData("5 IN (1, NULL)")] + [InlineData("5 NOT IN (1, NULL)")] + [InlineData("5 NOT IN (NULL)")] + public void In_is_null_when_the_value_is_or_a_miss_meets_a_null_item(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("'abc' IN (1)")] + [InlineData("1 IN ('abc')")] + [InlineData("5 IN ('abc', 5)")] + [InlineData("1 IN (G)")] + public void In_raises_a_type_mismatch_at_the_first_item_that_cannot_be_compared(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("5 IN (1, NULL)", 0)] + [InlineData("5 NOT IN (1, NULL)", 0)] + [InlineData("5 IN (5, NULL)", 1)] + [InlineData("5 NOT IN (5, NULL)", 0)] + public void A_where_condition_matches_no_row_on_an_unknown_in(string condition, int expected) => + Assert.Equal(expected, Count(condition)); + + [Theory] + [InlineData("NULL IS NULL", true)] + [InlineData("1 IS NULL", false)] + [InlineData("'' IS NULL", false)] + [InlineData("NT IS NULL", true)] + [InlineData("NT IS NOT NULL", false)] + [InlineData("NOT NT IS NULL", false)] + [InlineData("NOT NT IS NOT NULL", true)] + [InlineData("(NT) IS NULL", true)] + [InlineData("1 + NULL IS NULL", true)] + [InlineData("NULL & NULL IS NULL", true)] + [InlineData("1 IS NULL OR 1 = 1", true)] + [InlineData("G IS NULL", false)] + [InlineData("B IS NOT NULL", true)] + [InlineData("NT IS NULL IS NULL", false)] + public void Is_null_is_never_null_itself(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); +} diff --git a/test/LibRed.Engine.Tests/ProgramFlowFunctionTests.cs b/test/LibRed.Engine.Tests/ProgramFlowFunctionTests.cs new file mode 100644 index 000000000..389fd9465 --- /dev/null +++ b/test/LibRed.Engine.Tests/ProgramFlowFunctionTests.cs @@ -0,0 +1,221 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// IIf Choose Switch IsNumeric IsError TypeName VarType +/// Partition RGB QBColor. The expected values were measured against ACE, except that a Null +/// argument gives Null where ACE raises an error. +/// +public class ProgramFlowFunctionTests(ProgramFlowFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, BT BYTE, SG REAL, YN BIT, CY CURRENCY, DC DECIMAL(18,4), NT TEXT(60), DT DATETIME, G GUID, B BINARY(4))", + "INSERT INTO T (Id, BT, SG, YN, CY, DC, DT) VALUES (1, 1, 1.5, TRUE, 3.25, 4.5, #2020-01-02 12:00:00#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("flow-", Setup); + + private object? Scalar(string expression) => + database.Scalar($"SELECT {expression} FROM T", CultureInfo.GetCultureInfo("en-AU")); + + [Theory] + [InlineData("0.4", "F")] + [InlineData("-0.4", "F")] + [InlineData("0.5", "F")] + [InlineData("0.6", "T")] + [InlineData("-0.6", "T")] + [InlineData("1.5", "T")] + [InlineData("1E+20", "T")] + [InlineData("CSNG(0.1)", "F")] + [InlineData("'0.4'", "F")] + [InlineData("'0'", "F")] + [InlineData("'abc'", "T")] + [InlineData("'False'", "T")] + [InlineData("#1899-12-30 06:00#", "F")] + [InlineData("DT", "T")] + [InlineData("G", "T")] + [InlineData("NULL", "F")] + [InlineData("CDBL(0.4) AND TRUE", "T")] + public void Iif_rounds_its_condition_to_a_whole_number(string condition, string expected) => + Assert.Equal(expected, Scalar($"IIF({condition}, 'T', 'F')")); + + [Theory] + [InlineData("IIF(TRUE, 1, 1/0)", 1.0)] + [InlineData("IIF(FALSE, 1/0, 2)", 2.0)] + public void Iif_evaluates_only_the_branch_it_takes(string expression, double expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("0.4", "T")] + [InlineData("'True'", "T")] + [InlineData("'False'", "X")] + [InlineData("'0'", "X")] + [InlineData("'1'", "T")] + [InlineData("NULL", "X")] + [InlineData("DT", "T")] + [InlineData("CCUR(0.00001)", "X")] + public void Switch_reads_a_condition_as_cbool_does(string condition, string expected) => + Assert.Equal(expected, Scalar($"SWITCH({condition}, 'T', TRUE, 'X')")); + + [Theory] + [InlineData("'abc'")] + [InlineData("''")] + [InlineData("'yes'")] + [InlineData("G")] + [InlineData("B")] + public void Switch_condition_that_is_not_a_boolean_is_a_type_mismatch(string condition) => + Assert.Throws(() => Scalar($"SWITCH({condition}, 'T', TRUE, 'X')")); + + [Theory] + [InlineData("1", "a")] + [InlineData("1.5", "a")] + [InlineData("2.5", "b")] + [InlineData("2.9", "b")] + [InlineData("3.5", "c")] + [InlineData("'2.5'", "b")] + [InlineData("SG", "a")] + [InlineData("0.9", null)] + [InlineData("-0.5", null)] + [InlineData("TRUE", null)] + [InlineData("YN", null)] + [InlineData("4", null)] + [InlineData("DT", null)] + [InlineData("1E+20", null)] + public void Choose_truncates_its_index(string index, string? expected) => + Assert.Equal(expected, Scalar($"CHOOSE({index}, 'a', 'b', 'c')")); + + [Theory] + [InlineData("SWITCH(TRUE, 1, 1/0, 2)")] + [InlineData("SWITCH(FALSE, 1/0, TRUE, 2)")] + [InlineData("CHOOSE(1, 'a', 1/0)")] + [InlineData("CHOOSE(2, 1/0, 'b')")] + [InlineData("ISERROR(1/0)")] + public void Every_argument_is_evaluated(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("1", true)] + [InlineData("TRUE", true)] + [InlineData("YN", true)] + [InlineData("'1e3'", true)] + [InlineData("'&HFF'", true)] + [InlineData("'$5'", true)] + [InlineData("'1,000'", true)] + [InlineData("'1-'", true)] + [InlineData("'(1)'", true)] + [InlineData("'1d2'", true)] + [InlineData("'+1'", true)] + [InlineData("' 1 '", true)] + [InlineData("'-'", false)] + [InlineData("'.'", false)] + [InlineData("'1e400'", false)] + [InlineData("'Infinity'", false)] + [InlineData("'NaN'", false)] + [InlineData("'1 000'", false)] + [InlineData("'0x1F'", false)] + [InlineData("'True'", false)] + [InlineData("''", false)] + [InlineData("DT", false)] + [InlineData("G", false)] + [InlineData("B", false)] + [InlineData("NULL", false)] + public void Isnumeric_reads_text_as_a_number(string argument, bool expected) => + Assert.Equal(expected, Scalar($"ISNUMERIC({argument})")); + + [Theory] + [InlineData("TRUE", "Boolean", 11)] + [InlineData("YN", "Boolean", 11)] + [InlineData("BT", "Byte", 17)] + [InlineData("CBYTE(1)", "Byte", 17)] + [InlineData("CINT(1)", "Integer", 2)] + [InlineData("1", "Long", 3)] + [InlineData("Id", "Long", 3)] + [InlineData("SG", "Single", 4)] + [InlineData("CDBL(1)", "Double", 5)] + [InlineData("CCUR(1)", "Currency", 6)] + [InlineData("CY", "Currency", 6)] + [InlineData("CY * 2", "Currency", 6)] + [InlineData("CY + 1", "Currency", 6)] + [InlineData("DC", "Decimal", 14)] + [InlineData("DC * 1", "Decimal", 14)] + [InlineData("DC + CY", "Decimal", 14)] + [InlineData("DT", "Date", 7)] + [InlineData("'a'", "String", 8)] + [InlineData("G", "String", 8)] + [InlineData("B", "String", 8)] + [InlineData("NULL", "Null", 1)] + [InlineData("1/2", "Double", 5)] + public void Typename_and_vartype_use_the_access_names(string argument, string typeName, int varType) + { + Assert.Equal(typeName, Scalar($"TYPENAME({argument})")); + Assert.Equal(varType, Scalar($"VARTYPE({argument})")); + } + + [Theory] + [InlineData("PARTITION(-1, 0, 100, 10)", " : -1")] + [InlineData("PARTITION(5, 0, 100, 10)", " 0: 9")] + [InlineData("PARTITION(101, 0, 100, 10)", "101: ")] + [InlineData("PARTITION(9.5, 0, 100, 10)", " 10: 19")] + [InlineData("PARTITION('15', 0, 100, 10)", " 10: 19")] + [InlineData("PARTITION(DT, 0, 50000, 1000)", "43000:43999")] + public void Partition_labels_the_interval(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("RGB(255, 255, 255)", 16777215)] + [InlineData("RGB(256, 0, 0)", 255)] + [InlineData("RGB(0, 1000, 0)", 65280)] + [InlineData("RGB(1.5, 2.5, 0)", 514)] + [InlineData("RGB('10', 0, 0)", 10)] + [InlineData("RGB(0, 0, -0.5)", 0)] + [InlineData("RGB(255.5, 0, 0)", 255)] + [InlineData("QBCOLOR(7)", 12632256)] + [InlineData("QBCOLOR(15)", 16777215)] + [InlineData("QBCOLOR(1.5)", 32768)] + [InlineData("QBCOLOR('3')", 8421376)] + [InlineData("QBCOLOR(-0.5)", 0)] + public void Colours_read_their_parts_as_integers(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("PARTITION(5, 0, 100, 0)")] + [InlineData("PARTITION(5, 0, 100, -1)")] + [InlineData("PARTITION(5, 10, 10, 1)")] + [InlineData("PARTITION(5, 10, 5, 1)")] + [InlineData("PARTITION(5, -1, 100, 10)")] + [InlineData("RGB(-1, 0, 0)")] + [InlineData("RGB(TRUE, 0, 0)")] + [InlineData("RGB(0, 0, -0.6)")] + [InlineData("QBCOLOR(16)")] + [InlineData("QBCOLOR(-1)")] + [InlineData("QBCOLOR(TRUE)")] + [InlineData("QBCOLOR(15.5)")] + public void Out_of_range_arguments_are_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("RGB(0, 0, 40000)")] + [InlineData("RGB(0, 0, 2147483648)")] + public void Colour_parts_past_an_integer_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("PARTITION(NULL, 0, 100, 10)")] + [InlineData("PARTITION(5, NULL, 100, 10)")] + [InlineData("PARTITION(5, 0, NULL, 10)")] + [InlineData("PARTITION(5, 0, 100, NULL)")] + [InlineData("RGB(NULL, 0, 0)")] + [InlineData("QBCOLOR(NULL)")] + [InlineData("CHOOSE(1, NULL, 'b')")] + [InlineData("SWITCH(FALSE, 1)")] + [InlineData("IIF(FALSE, 1)")] + public void Null_results(string expression) => + Assert.Null(Scalar(expression)); +} diff --git a/test/LibRed.Engine.Tests/ReferencesAndIdentityTests.cs b/test/LibRed.Engine.Tests/ReferencesAndIdentityTests.cs new file mode 100644 index 000000000..268977f39 --- /dev/null +++ b/test/LibRed.Engine.Tests/ReferencesAndIdentityTests.cs @@ -0,0 +1,242 @@ +using LibRed; +using LibRed.Catalog; +using LibRed.Engine; +using LibRed.Sql.Ast; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// REFERENCES table with no column list, and the trailing IDENTITY [(seed [, increment])] column +/// attribute — both measured against ACE (see ReferencesAndIdentityAccessTests, which runs each statement +/// through both engines). +/// +public class ReferencesAndIdentityTests +{ + private const string Parent = "CREATE TABLE P (Id LONG CONSTRAINT pkP PRIMARY KEY, Code TEXT(10))"; + + private static SqlStatement Parse(string sql) => new AntlrSqlParser().ParseStatement(sql); + + private static JetDatabase Run(params string[] statements) + { + string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "refident-"); + JetDatabase db = TemporaryDatabase.OpenTracked(path, readOnly: false); + var engine = new QueryEngine(db); + foreach (string statement in statements) engine.ExecuteNonQuery(statement); + return db; + } + + // ---- REFERENCES table, no column list ---- + + [Fact] + public void References_without_columns_pairs_with_the_parent_primary_key_by_position() + { + // The child column shares its name with a non-key parent column, and still references the key. + JetDatabase db = Run(Parent, "CREATE TABLE C (Id LONG, Code LONG, CONSTRAINT fk FOREIGN KEY (Code) REFERENCES P)"); + ForeignKey fk = Assert.Single(db.Catalog.ForeignKeysOf("C")); + Assert.Equal("P", fk.ReferencedTable); + Assert.Equal([("Code", "Id")], fk.Columns); + } + + [Fact] + public void A_composite_key_pairs_in_key_order() + { + JetDatabase db = Run( + "CREATE TABLE P2 (A LONG, B LONG, CONSTRAINT pkP2 PRIMARY KEY (A, B))", + "CREATE TABLE C (Id LONG, X LONG, Y LONG, CONSTRAINT fk FOREIGN KEY (X, Y) REFERENCES P2)"); + Assert.Equal([("X", "A"), ("Y", "B")], Assert.Single(db.Catalog.ForeignKeysOf("C")).Columns); + } + + [Theory] + [InlineData("CREATE TABLE P3 (Id LONG, Code TEXT(10) CONSTRAINT uq3 UNIQUE)", "CREATE TABLE C (Id LONG, PId LONG REFERENCES P3)", "does not have a primary key")] + [InlineData("CREATE TABLE P2 (A LONG, B LONG, CONSTRAINT pkP2 PRIMARY KEY (A, B))", "CREATE TABLE C (Id LONG, X LONG, CONSTRAINT fk FOREIGN KEY (X) REFERENCES P2)", "same number of fields")] + [InlineData("CREATE TABLE Unrelated (Id LONG)", "CREATE TABLE C (Id LONG, PId LONG REFERENCES Nope)", "Cannot find table")] + public void Ace_refuses_it_and_so_does_libred(string parent, string child, string message) + { + var error = Assert.Throws(() => Run(parent, child)); + Assert.Contains(message, error.Message); + } + + [Fact] + public void A_self_reference_takes_a_primary_key_declared_before_it() + { + JetDatabase db = Run("CREATE TABLE C (Id LONG CONSTRAINT pkC PRIMARY KEY, ParentId LONG REFERENCES C)"); + Assert.Equal([("ParentId", "Id")], Assert.Single(db.Catalog.ForeignKeysOf("C")).Columns); + } + + [Theory] + [InlineData("CREATE TABLE C (ParentId LONG REFERENCES C, Id LONG CONSTRAINT pkC PRIMARY KEY)")] + [InlineData("CREATE TABLE C (Id LONG, ParentId LONG, CONSTRAINT fk FOREIGN KEY (ParentId) REFERENCES C, CONSTRAINT pkC PRIMARY KEY (Id))")] + public void A_self_reference_before_its_primary_key_has_no_key_to_reference(string sql) + { + var error = Assert.Throws(() => Run(sql)); + Assert.Contains("does not have a primary key", error.Message); + } + + [Theory] + [InlineData("ALTER TABLE C ADD COLUMN PId LONG REFERENCES P")] + [InlineData("ALTER TABLE C ADD COLUMN PId LONG CONSTRAINT fk REFERENCES P (Id)")] + [InlineData("ALTER TABLE C ADD CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P")] + public void Alter_table_creates_the_relationship(string alter) + { + string create = alter.Contains("ADD COLUMN") ? "CREATE TABLE C (Id LONG)" : "CREATE TABLE C (Id LONG, PId LONG)"; + JetDatabase db = Run(Parent, create, alter); + Assert.Equal([("PId", "Id")], Assert.Single(db.Catalog.ForeignKeysOf("C")).Columns); + } + + [Theory] + [InlineData("COUNTER", "LONG")] + [InlineData("LONG", "COUNTER")] + [InlineData("TEXT(10)", "TEXT(20)")] + [InlineData("CHAR(10)", "TEXT(5)")] + [InlineData("DECIMAL(10, 2)", "DECIMAL(12, 4)")] + [InlineData("BINARY(8)", "VARBINARY(8)")] + public void A_relationship_pairs_columns_of_the_same_storage_type_whatever_their_lengths(string parent, string child) + { + JetDatabase db = Run($"CREATE TABLE P (Id {parent} CONSTRAINT pkP PRIMARY KEY)", $"CREATE TABLE C (K LONG, PId {child} REFERENCES P (Id))"); + Assert.Single(db.Catalog.ForeignKeysOf("C")); + } + + [Theory] + [InlineData("LONG", "TEXT(10)", false)] + [InlineData("LONG", "SHORT", false)] + [InlineData("DOUBLE", "SINGLE", false)] + [InlineData("CURRENCY", "DECIMAL(10, 2)", false)] + [InlineData("LONG", "TEXT(10)", true)] + public void A_relationship_between_different_storage_types_is_refused(string parent, string child, bool alterTable) + { + string[] statements = alterTable + ? [$"CREATE TABLE P (Id {parent} CONSTRAINT pkP PRIMARY KEY)", $"CREATE TABLE C (K LONG, PId {child})", "ALTER TABLE C ADD CONSTRAINT fk FOREIGN KEY (PId) REFERENCES P (Id)"] + : [$"CREATE TABLE P (Id {parent} CONSTRAINT pkP PRIMARY KEY)", $"CREATE TABLE C (K LONG, PId {child} REFERENCES P (Id))"]; + var error = Assert.Throws(() => Run(statements)); + Assert.Contains("same data types", error.Message); + } + + // ---- ADD COLUMN constraints ---- + + [Fact] + public void Add_column_creates_its_primary_key_and_unique_index() + { + JetDatabase db = Run("CREATE TABLE T (V TEXT(10))", + "ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY", + "ALTER TABLE T ADD COLUMN Code TEXT(10) CONSTRAINT uq UNIQUE"); + TableDef t = db.Catalog.FindTable("T")!; + IndexDef pk = Assert.Single(t.Indexes, i => i.IsPrimaryKey); + Assert.Equal("pk", pk.Name); + IndexDef uq = Assert.Single(t.Indexes, i => i.Name == "uq"); + Assert.True(uq.IsUnique); + Assert.Equal("Code", Assert.Single(uq.Columns).Column.Name); + } + + [Theory] + [InlineData("CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, V TEXT(10));ALTER TABLE T ADD COLUMN Id2 LONG CONSTRAINT pk2 PRIMARY KEY", "Primary key already exists")] + [InlineData("CREATE TABLE T (Id LONG CONSTRAINT pk PRIMARY KEY, W LONG);ALTER TABLE T ADD CONSTRAINT pk2 PRIMARY KEY (W)", "Primary key already exists")] + [InlineData("CREATE TABLE T (V TEXT(10));INSERT INTO T (V) VALUES ('a');ALTER TABLE T ADD COLUMN Id LONG CONSTRAINT pk PRIMARY KEY", "cannot contain a Null value")] + [InlineData("CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX pk ON T (W) WITH PRIMARY", "cannot contain a Null value")] + [InlineData("CREATE TABLE T (V TEXT(10), W LONG);INSERT INTO T (V) VALUES ('a');CREATE INDEX ix ON T (V, W) WITH DISALLOW NULL", "cannot contain a Null value")] + public void A_second_primary_key_or_a_key_over_nulls_is_refused(string statements, string message) + { + var error = Assert.Throws(() => Run(statements.Split(';'))); + Assert.Contains(message, error.Message); + } + + [Theory] + [InlineData("COUNTER", "1,2,3,4")] + [InlineData("COUNTER(1, 1)", "1,2,3,4")] + [InlineData("INT IDENTITY", "1,2,3,4")] + [InlineData("COUNTER(2, 1)", "1,2,2,3")] + [InlineData("COUNTER(1, 5)", "1,1,2,6")] + [InlineData("COUNTER(10, -1)", "1,2,9,10")] + public void An_autonumber_added_to_a_populated_table_numbers_its_rows(string type, string ids) + { + JetDatabase db = Run("CREATE TABLE T (V TEXT(10))", "INSERT INTO T (V) VALUES ('a')", "INSERT INTO T (V) VALUES ('b')", + $"ALTER TABLE T ADD COLUMN Id {type}"); + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("INSERT INTO T (V) VALUES ('c')"); + engine.ExecuteNonQuery("INSERT INTO T (V) VALUES ('d')"); + Assert.Equal(ids, string.Join(",", engine.ExecuteQuery("SELECT Id FROM T ORDER BY Id").Rows.Select(r => r[0]))); + } + + // ---- IDENTITY ---- + + [Theory] + [InlineData("Id INT NOT NULL IDENTITY", 1, 1, true)] + [InlineData("Id INT IDENTITY NOT NULL", 1, 1, true)] + [InlineData("Id INT IDENTITY", 1, 1, false)] + [InlineData("Id INT NOT NULL IDENTITY(5, 2)", 5, 2, true)] + [InlineData("Id INT IDENTITY(5)", 5, 1, false)] + [InlineData("Id LONG NOT NULL IDENTITY(5, -1)", 5, -1, true)] + [InlineData("Id IDENTITY(5, 2)", 5, 2, false)] + [InlineData("Id INT IDENTITY DEFAULT 1", 1, 1, false)] + [InlineData("Id INT NOT NULL IDENTITY PRIMARY KEY", 1, 1, true)] + [InlineData("Id COUNTER(5, 2) IDENTITY(9, 3)", 9, 3, false)] + // IDENTITY's own seed and increment — 1 each when omitted — replace the type's. + [InlineData("Id COUNTER(5, 2) IDENTITY", 1, 1, false)] + public void Identity_makes_a_long_column_an_autonumber(string column, int seed, int increment, bool required) + { + JetDatabase db = Run($"CREATE TABLE T ({column}, V TEXT(10))"); + ColumnDef id = db.Catalog.FindTable("T")!.FindColumn("Id")!; + Assert.True(id.IsAutoNumber); + Assert.Equal(increment, id.Increment); + Assert.Equal(!required, id.IsNullable); + + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("INSERT INTO T (V) VALUES ('a')"); + Assert.Equal(seed, Convert.ToInt32(engine.ExecuteQuery("SELECT Id FROM T").Rows.Single()[0])); + } + + [Theory] + [InlineData("Id SHORT NOT NULL IDENTITY", JetDataType.Int16)] + [InlineData("Id BYTE IDENTITY(5, 2)", JetDataType.Byte)] + [InlineData("Id BIGINT IDENTITY", JetDataType.Int64)] + [InlineData("Id DOUBLE IDENTITY", JetDataType.Double)] + [InlineData("Id GUID IDENTITY", JetDataType.Guid)] + [InlineData("Id TEXT(10) NOT NULL IDENTITY", JetDataType.Text)] + [InlineData("Id TEXT IDENTITY(5, 2)", JetDataType.Memo)] + public void Identity_on_any_other_type_is_ignored(string column, JetDataType type) + { + ColumnDef id = Run($"CREATE TABLE T ({column}, V TEXT(10))").Catalog.FindTable("T")!.FindColumn("Id")!; + Assert.Equal(type, id.Type); + Assert.False(id.IsAutoNumber); + } + + [Theory] + [InlineData("Id INT PRIMARY KEY IDENTITY(5, 2)")] + [InlineData("Id INT CONSTRAINT pk PRIMARY KEY NOT NULL IDENTITY")] + [InlineData("Id INT DEFAULT 1 IDENTITY")] + [InlineData("Id INT NOT NULL IDENTITY(5, 2, 1)")] + [InlineData("Id INT NOT NULL IDENTITY()")] + // COUNTER and AUTOINCREMENT are types only; neither trails a type as IDENTITY does. + [InlineData("Id INT NOT NULL AUTOINCREMENT")] + [InlineData("Id INT NOT NULL COUNTER")] + // Reserved, as ACE reserves it. + [InlineData("Identity LONG")] + public void Ace_refuses_these_and_so_does_libred(string column) + => Assert.ThrowsAny(() => Parse($"CREATE TABLE T ({column}, V TEXT(10))")); + + [Fact] + public void A_bracketed_column_named_identity_is_still_allowed() + => Assert.Equal("Identity", Assert.IsType(Parse("CREATE TABLE T ([Identity] LONG)")).Columns[0].Name); + + [Fact] + public void At_at_identity_still_reads_the_last_autonumber() + { + JetDatabase db = Run("CREATE TABLE T (Id INT NOT NULL IDENTITY(7, 1), V TEXT(10))"); + var engine = new QueryEngine(db); + engine.ExecuteNonQuery("INSERT INTO T (V) VALUES ('a')"); + Assert.Equal(7, Convert.ToInt32(engine.ExecuteQuery("SELECT @@IDENTITY").Rows.Single()[0])); + } + + [Fact] + public void Alter_table_takes_identity_too() + { + JetDatabase db = Run("CREATE TABLE T (V TEXT(10))", "ALTER TABLE T ADD COLUMN Id INT NOT NULL IDENTITY(3, 3)"); + ColumnDef id = db.Catalog.FindTable("T")!.FindColumn("Id")!; + Assert.True(id.IsAutoNumber); + Assert.False(id.IsNullable); + + db = Run("CREATE TABLE T (Id INT, V TEXT(10))", "ALTER TABLE T ALTER COLUMN Id INT IDENTITY"); + Assert.True(db.Catalog.FindTable("T")!.FindColumn("Id")!.IsAutoNumber); + } +} diff --git a/test/LibRed.Engine.Tests/RegressionAggregateTests.cs b/test/LibRed.Engine.Tests/RegressionAggregateTests.cs new file mode 100644 index 000000000..bfc27e386 --- /dev/null +++ b/test/LibRed.Engine.Tests/RegressionAggregateTests.cs @@ -0,0 +1,116 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The standard's binary set functions — CORR, COVAR_POP, COVAR_SAMP and the REGR_ family — and its names for the +/// statistical aggregates (STDDEV_SAMP, STDDEV_POP, VAR_SAMP, VAR_POP), grouped and over a window. Access has none of +/// them; this is a LibRed extension. +/// +public class RegressionAggregateTests(RegressionAggregateTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Group a: (x, y) = (1, 2), (2, 4), (3, 5), (4, 4), (5, 5), so Sxx = 10, Syy = 6, Sxy = 6 — plus a pair missing y + // and one missing x, which take no part. Group b has every x the same, c every y, and d a single pair. + private static readonly string[] Setup = + [ + "CREATE TABLE R (Id LONG, G TEXT(10), Y DOUBLE, X LONG, M CURRENCY)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (1, 'a', 2, 1, 1.25)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (2, 'a', 4, 2, 2.5)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (3, 'a', 5, 3, 3.75)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (4, 'a', 4, 4, 1)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (5, 'a', 5, 5, 2)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (6, 'a', NULL, 6, 3)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (7, 'a', 7, NULL, 4)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (8, 'b', 3, 1, 5)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (9, 'b', 5, 1, 6)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (10, 'c', 2, 1, 7)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (11, 'c', 2, 2, 8)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (12, 'c', 2, 3, 9)", + "INSERT INTO R (Id, G, Y, X, M) VALUES (13, 'd', 1, 5, 10)", + ]; + + public sealed class Database() : SharedDatabase("regression-", Setup); + + private (IReadOnlyList Types, List Rows) Query(string sql) => + database.Query(sql, CultureInfo.InvariantCulture); + + // Rounded to ten places: the sums are exact for this data, the divisions not always. + private static string Format(object? value) => + value is double d ? Math.Round(d, 10).ToString(CultureInfo.InvariantCulture) : Convert.ToString(value, CultureInfo.InvariantCulture)!; + + private string ByGroup(string aggregate) => string.Join(" ", + Query($"SELECT G, {aggregate} AS r FROM R GROUP BY G ORDER BY G").Rows.Select(row => $"{row[0]}:{Format(row[1])}")); + + [Theory] + [InlineData("REGR_COUNT", "a:5 b:2 c:3 d:1")] + [InlineData("REGR_AVGX", "a:3 b:1 c:2 d:5")] + [InlineData("REGR_AVGY", "a:4 b:4 c:2 d:1")] + [InlineData("REGR_SXX", "a:10 b:0 c:2 d:0")] + [InlineData("REGR_SYY", "a:6 b:2 c:0 d:0")] + [InlineData("REGR_SXY", "a:6 b:0 c:0 d:0")] + [InlineData("REGR_SLOPE", "a:0.6 b: c:0 d:")] + [InlineData("REGR_INTERCEPT", "a:2.2 b: c:2 d:")] + [InlineData("REGR_R2", "a:0.6 b: c:1 d:")] + [InlineData("COVAR_POP", "a:1.2 b:0 c:0 d:0")] + [InlineData("COVAR_SAMP", "a:1.5 b:0 c:0 d:")] + [InlineData("CORR", "a:0.7745966692 b: c: d:")] + public void A_binary_set_function_reads_the_pairs_of_the_group(string function, string expected) => + Assert.Equal(expected, ByGroup($"{function}(Y, X)")); + + [Theory] + [InlineData("REGR_COUNT", 0)] + [InlineData("REGR_SLOPE", null)] + [InlineData("CORR", null)] + [InlineData("COVAR_POP", null)] + public void Over_no_pairs_only_the_count_has_a_value(string function, object? expected) => + Assert.Equal(expected, database.Scalar($"SELECT {function}(Y, X) FROM R WHERE Id > 100", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("REGR_COUNT", typeof(int))] + [InlineData("REGR_SLOPE", typeof(double))] + [InlineData("CORR", typeof(double))] + [InlineData("COVAR_SAMP", typeof(double))] + public void Over_a_partition_it_is_the_grouped_function_of_that_group(string function, Type expected) + { + var grouped = Query($"SELECT G, {function}(Y, X) FROM R GROUP BY G").Rows.ToDictionary(row => (string)row[0]!, row => row[1]); + var (types, rows) = Query($"SELECT G, {function}(Y, X) OVER (PARTITION BY G) FROM R"); + + Assert.Equal(expected, types[1]); + Assert.All(rows, row => Assert.Equal(grouped[(string)row[0]!], row[1])); + } + + [Fact] + public void Over_a_window_it_runs_through_the_frame() => + Assert.Equal("1:1 2:2 3:3 4:4 5:5 6:5 7:5", string.Join(" ", + Query("SELECT Id, REGR_COUNT(Y, X) OVER (ORDER BY Id) FROM R WHERE G = 'a' ORDER BY Id").Rows + .Select(row => $"{row[0]}:{row[1]}"))); + + [Theory] + [InlineData("SELECT CORR(Y) FROM R")] + [InlineData("SELECT CORR(Y, X, M) FROM R")] + [InlineData("SELECT CORR(DISTINCT Y, X) OVER () FROM R")] + public void The_pair_is_required_and_takes_no_distinct(string sql) => + Assert.Throws(() => Query(sql)); + + [Fact] + public void A_grouped_one_takes_no_distinct_either() => + Assert.Throws(() => Query("SELECT CORR(DISTINCT Y, X) FROM R")); + + [Theory] + [InlineData("STDDEV_SAMP", "STDEV")] + [InlineData("STDDEV_POP", "STDEVP")] + [InlineData("VAR_SAMP", "VAR")] + [InlineData("VAR_POP", "VARP")] + public void The_standard_names_are_the_access_statistics(string standard, string access) + { + foreach (string argument in new[] { "Y", "X", "M" }) + { + Assert.Equal(Query($"SELECT G, {access}({argument}) FROM R GROUP BY G").Rows, Query($"SELECT G, {standard}({argument}) FROM R GROUP BY G").Rows); + Assert.Equal( + Query($"SELECT Id, {access}({argument}) OVER (ORDER BY Id) FROM R").Rows, + Query($"SELECT Id, {standard}({argument}) OVER (ORDER BY Id) FROM R").Rows); + } + } +} diff --git a/test/LibRed.Engine.Tests/ResultColumnTypeTests.cs b/test/LibRed.Engine.Tests/ResultColumnTypeTests.cs new file mode 100644 index 000000000..1147d14ef --- /dev/null +++ b/test/LibRed.Engine.Tests/ResultColumnTypeTests.cs @@ -0,0 +1,142 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// An expression that chooses among values — CASE, IIF, COALESCE — and a set operation declare one type for their +/// column, and every value comes back as that type. CASE, IIF and COALESCE widen numbers on one ladder; a UNION +/// types its columns as ACE does (verified vs ACE), including its binary and text columns for mixed kinds. +/// +public class ResultColumnTypeTests(ResultColumnTypeTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id INT, M CURRENCY, B BYTE, S SMALLINT, D DATETIME, F DOUBLE, R REAL, E DECIMAL(18,4), " + + "L LONG, X TEXT(10), Y YESNO, G GUID, N BINARY(4))", + "INSERT INTO T (Id, M, B, S, D, F, R, E, L, X, Y) " + + "VALUES (1, 10.5, 3, 7, #2020-01-02#, 1.5, 1.5, 4.25, 70000, 'abc', TRUE)", + "INSERT INTO T (Id, M, B, S, D, F, R, E, L, X, Y) " + + "VALUES (2, 2, 4, 8, #1899-12-30 06:00:00#, 2.5, 0.75, 5.5, 200000, 'xyz', FALSE)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET N = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("result-types-", Setup); + + /// The query's last column: its declared type, and its values, which must all be of that type. + private (Type Declared, object?[] Values) Column(string sql) + { + var (types, rows) = database.Query(sql, CultureInfo.GetCultureInfo("en-US")); + Type declared = types[^1]; + object?[] values = rows.Select(row => row[^1]).ToArray(); + Assert.All(values, value => Assert.True(value is null || value.GetType() == declared, + $"{value} is a {value?.GetType().Name}, declared {declared.Name}")); + return (declared, values); + } + + private static string Union(string left, string right) => + $"SELECT {left} AS c FROM T WHERE Id = 1 UNION ALL SELECT {right} FROM T WHERE Id = 1"; + + [Theory] + // A number written with a decimal point is a Decimal to the ladder, as ACE reads it, so money with one stays money. + [InlineData("CASE WHEN M >= 5.1 THEN M ELSE 5.1 END", typeof(decimal))] + [InlineData("IIF(M >= 5.1, M, 5.1)", typeof(decimal))] + [InlineData("IIF(Id = 1, 0.5, Id)", typeof(decimal))] + [InlineData("IIF(Id = 1, 0.5, F)", typeof(double))] // a Double still wins + [InlineData("CASE WHEN Id = 1 THEN Id ELSE F END", typeof(double))] + [InlineData("IIF(Id = 1, B, S)", typeof(short))] + [InlineData("IIF(Id = 1, B, 5)", typeof(int))] + [InlineData("IIF(Id = 1, R, S)", typeof(float))] + [InlineData("IIF(Id = 1, R, L)", typeof(double))] + [InlineData("IIF(Id = 1, M, L)", typeof(decimal))] + [InlineData("IIF(Id = 1, NULL, B)", typeof(byte))] + [InlineData("COALESCE(NULL, B, 5)", typeof(int))] + [InlineData("CASE WHEN Id = 1 THEN NULL ELSE S END", typeof(short))] + public void A_choice_of_numbers_widens_on_one_ladder(string expression, Type expected) => + Assert.Equal(expected, Column($"SELECT Id, {expression} AS c FROM T ORDER BY Id").Declared); + + [Theory] + [InlineData("B", "5", typeof(int))] + [InlineData("5", "B", typeof(int))] + [InlineData("S", "70000", typeof(int))] + [InlineData("B", "S", typeof(short))] + [InlineData("NULL", "B", typeof(byte))] + [InlineData("B", "NULL", typeof(byte))] + [InlineData("M", "F", typeof(double))] + [InlineData("E", "F", typeof(double))] + [InlineData("M", "E", typeof(decimal))] + [InlineData("M", "L", typeof(decimal))] + [InlineData("R", "S", typeof(float))] + [InlineData("R", "L", typeof(double))] + [InlineData("Y", "B", typeof(short))] + [InlineData("Y", "F", typeof(double))] + [InlineData("Y", "Y", typeof(bool))] + [InlineData("D", "D", typeof(DateTime))] + [InlineData("G", "G", typeof(Guid))] + public void A_union_types_its_columns_as_ace_does(string left, string right, Type expected) => + Assert.Equal(expected, Column(Union(left, right)).Declared); + + [Fact] + public void A_union_reads_a_boolean_as_minus_one() + { + (_, object?[] values) = Column(Union("Y", "B")); + Assert.Equal(new object?[] { (short)-1, (short)3 }, values); + } + + [Theory] + [InlineData("X", "B", "abc", "3")] + [InlineData("Y", "X", "-1", "abc")] + [InlineData("D", "F", "1/2/2020", "1.5")] + [InlineData("D", "X", "1/2/2020", "abc")] + [InlineData("Y", "D", "-1", "1/2/2020")] + public void A_union_of_text_or_a_date_with_another_kind_is_text(string left, string right, string first, string second) + { + (Type declared, object?[] values) = Column(Union(left, right)); + Assert.Equal(typeof(string), declared); + Assert.Equal(new object?[] { first, second }, values); + } + + [Theory] + [InlineData("G", "X", "33221100554477668899AABBCCDDEEFF", "610062006300")] + [InlineData("G", "B", "33221100554477668899AABBCCDDEEFF", "03000000")] + [InlineData("N", "S", "41004200", "0700")] + [InlineData("N", "L", "41004200", "70110100")] + [InlineData("N", "F", "41004200", "000000000000F83F")] + [InlineData("N", "R", "41004200", "0000C03F")] + [InlineData("N", "D", "41004200", "000000000067E540")] + [InlineData("N", "Y", "41004200", "FFFF")] + [InlineData("N", "M", "41004200", "289A010000000000")] + [InlineData("N", "E", "41004200", "34002E0032003500")] + public void A_union_of_binary_or_a_guid_with_another_kind_is_binary(string left, string right, string first, string second) + { + (Type declared, object?[] values) = Column(Union(left, right)); + Assert.Equal(typeof(byte[]), declared); + Assert.Equal([first, second], values.Select(v => Convert.ToHexString((byte[])v!))); + } + + [Fact] + public void A_union_compares_values_after_converting_them() + { + (_, object?[] values) = Column("SELECT B AS c FROM T WHERE Id = 1 UNION SELECT 3 FROM T WHERE Id = 1"); + Assert.Equal(new object?[] { 3 }, values); + } + + // EF's Sum with a default writes the decimal zero as 0.0; the money sum stays a Decimal, to the last place. + [Fact] + public void A_money_sum_with_a_written_zero_stays_money() + { + var (types, rows) = database.Query("SELECT IIF(SUM(M) IS NULL, 0.0, SUM(M)) AS c FROM T", CultureInfo.GetCultureInfo("en-US")); + Assert.Equal(typeof(decimal), types[0]); + Assert.IsType(rows.Single()[0]); + } + + [Fact] + public void Values_keep_their_widened_value() + { + // Money with a written decimal is a Decimal, and the literal's value is the one written. + (_, object?[] values) = Column("SELECT Id, IIF(M >= 5.1, M, 5.1) AS c FROM T ORDER BY Id"); + Assert.Equal(new object?[] { 10.5m, 5.1m }, values); + } +} diff --git a/test/LibRed.Engine.Tests/SharedDatabase.cs b/test/LibRed.Engine.Tests/SharedDatabase.cs new file mode 100644 index 000000000..7f5c7231e --- /dev/null +++ b/test/LibRed.Engine.Tests/SharedDatabase.cs @@ -0,0 +1,72 @@ +using System.Globalization; +using LibRed.Engine; + +namespace LibRed.Engine.Tests; + +/// +/// One Northwind copy for a whole test class, set up once, for classes whose tests only read — the function and +/// operator tables, where every case is a single SELECT. A class takes it as an xunit class fixture through a nested +/// sealed subclass that supplies its setup; xunit disposes it, closing and deleting the copy, when the class is done. +/// A test that writes uses for a copy of its own instead. +/// +public abstract class SharedDatabase : IDisposable +{ + private static string Northwind => Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"); + + private readonly TemporaryDatabase _copy; + + protected SharedDatabase(string prefix, IReadOnlyList setup) + { + _copy = TemporaryDatabase.CopyOf(Northwind, prefix); + Engine = Prepare(new QueryEngine(_copy.Open()), setup); + } + + public QueryEngine Engine { get; } + + /// A copy of its own with the same setup, released when the calling test ends. + public static QueryEngine Fresh(string prefix, IReadOnlyList setup) => + Prepare(new QueryEngine(TemporaryDatabase.OpenTracked(TemporaryDatabase.CopyPath(Northwind, prefix))), setup); + + /// The first value of the query's first row, run under . The evaluator reads and + /// writes text in the regional format, so a test pins the culture whatever the machine's is. + public object? Scalar(string sql, CultureInfo culture) => Scalar(Engine, sql, culture); + + /// + public static object? Scalar(QueryEngine engine, string sql, CultureInfo culture) => + Under(culture, () => engine.ExecuteQuery(sql).Rows.First()[0]); + + /// The query's declared column types and all its rows, run under . + public (IReadOnlyList ColumnTypes, List Rows) Query(string sql, CultureInfo culture) => + Under(culture, () => + { + var result = Engine.ExecuteQuery(sql); + return (result.ColumnTypes, result.Rows.ToList()); + }); + + private static T Under(CultureInfo culture, Func run) + { + CultureInfo previous = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = culture; + try + { + return run(); + } + finally + { + CultureInfo.CurrentCulture = previous; + } + } + + private static QueryEngine Prepare(QueryEngine engine, IReadOnlyList setup) + { + foreach (string statement in setup) + engine.ExecuteNonQuery(statement); + return engine; + } + + public void Dispose() + { + _copy.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/test/LibRed.Engine.Tests/SqlQueryTests.cs b/test/LibRed.Engine.Tests/SqlQueryTests.cs index 0e192a4dd..ca190fa54 100644 --- a/test/LibRed.Engine.Tests/SqlQueryTests.cs +++ b/test/LibRed.Engine.Tests/SqlQueryTests.cs @@ -93,7 +93,7 @@ public void Null_concat_and_like_wildcards_follow_access_style() var rs = Query( "SELECT CustomerID, Region & '-' & City AS RegionCity " + "FROM Customers " + - "WHERE CustomerID LIKE 'A????' OR CustomerID LIKE 'B*' OR CustomerID = 'QUEDE' " + + "WHERE CustomerID LIKE 'A____' OR CustomerID LIKE 'B%' OR CustomerID = 'QUEDE' " + "ORDER BY CustomerID"); Assert.Equal(["CustomerID", "RegionCity"], rs.ColumnNames); diff --git a/test/LibRed.Engine.Tests/StandardMathFunctionTests.cs b/test/LibRed.Engine.Tests/StandardMathFunctionTests.cs new file mode 100644 index 000000000..8040ec469 --- /dev/null +++ b/test/LibRed.Engine.Tests/StandardMathFunctionTests.cs @@ -0,0 +1,129 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// The standard SQL math functions: Floor, Ceiling, Sign, Sqrt, Ln, Log10, two-argument Log, Power, the inverse and +/// hyperbolic trigonometric functions, Degrees, Radians and Pi. Access has none of them; they are LibRed extensions. +/// +public class StandardMathFunctionTests(StandardMathFunctionTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly string[] Setup = + [ + "CREATE TABLE M (Id LONG, DB FLOAT, SG REAL, CY CURRENCY, DC DECIMAL(18,4), LG LONG, DT DATETIME)", + "INSERT INTO M (Id, DB, SG, CY, DC, LG, DT) VALUES (1, -2.5, 1.5, -1.25, 2.75, -7, #2020-01-02 18:00#)", + "INSERT INTO M (Id) VALUES (2)", + ]; + + public sealed class Database() : SharedDatabase("standard-math-", Setup); + + private object? Scalar(string expression, int id = 1) => + database.Scalar($"SELECT {expression} FROM M WHERE Id = {id}", CultureInfo.InvariantCulture); + + private Type ColumnType(string expression) => + database.Query($"SELECT {expression} FROM M", CultureInfo.InvariantCulture).ColumnTypes[0]; + + [Theory] + [InlineData("FLOOR(DB)", -3d)] + [InlineData("CEILING(DB)", -2d)] + [InlineData("CEIL(2.5)", 3d)] + [InlineData("SQRT(6.25)", 2.5)] + [InlineData("LN(1)", 0d)] + [InlineData("LOG10(1000)", 3d)] + [InlineData("LOG(2, 8)", 3d)] + [InlineData("POWER(2, 10)", 1024d)] + [InlineData("POWER(DB, 2)", 6.25)] + [InlineData("ASIN(1)", Math.PI / 2)] + [InlineData("ACOS(-1)", Math.PI)] + [InlineData("ATAN(1)", Math.PI / 4)] + [InlineData("ATAN2(1, -1)", 3 * Math.PI / 4)] + [InlineData("ATAN2(-1, 0)", -Math.PI / 2)] + [InlineData("ATAN2(0, 0)", 0d)] + [InlineData("SINH(1)", 1.1752011936438014)] + [InlineData("COSH(0)", 1d)] + [InlineData("TANH(0.5)", 0.46211715726000974)] + [InlineData("DEGREES(PI())", 180d)] + [InlineData("RADIANS(180)", Math.PI)] + [InlineData("PI()", Math.PI)] + public void A_function_gives_its_value(string expression, double expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)), 1e-12); + + [Theory] + [InlineData("SIGN(DB)", -1)] + [InlineData("SIGN(0)", 0)] + [InlineData("SIGN(LG)", -1)] + [InlineData("SIGN('3')", 1)] + public void Sign_is_sgn(string expression, int expected) => Assert.Equal(expected, Scalar(expression)); + + [Fact] + public void Floor_and_ceiling_keep_the_operands_type() + { + Assert.Equal(-2m, Scalar("FLOOR(CY)")); + Assert.Equal(3m, Scalar("CEILING(DC)")); + Assert.Equal(-7, Scalar("FLOOR(LG)")); + Assert.Equal(2f, Scalar("CEILING(SG)")); + Assert.Equal(new DateTime(2020, 1, 2), Scalar("FLOOR(DT)")); + Assert.Equal(new DateTime(2020, 1, 3), Scalar("CEILING(DT)")); + } + + [Theory] + [InlineData("FLOOR(CY)", typeof(decimal))] + [InlineData("CEILING(SG)", typeof(float))] + [InlineData("FLOOR(LG)", typeof(int))] + [InlineData("CEIL(DT)", typeof(DateTime))] + [InlineData("SIGN(DB)", typeof(int))] + [InlineData("SQRT(DB * DB)", typeof(double))] + [InlineData("LOG(10, LG * LG)", typeof(double))] + [InlineData("POWER(LG, 2)", typeof(double))] + [InlineData("ATAN2(DB, LG)", typeof(double))] + [InlineData("TANH(CY)", typeof(double))] + [InlineData("PI()", typeof(double))] + public void The_column_is_declared_as_the_value_it_returns(string expression, Type expected) + { + Assert.Equal(expected, ColumnType(expression)); + Assert.True(Scalar(expression) is var value && value?.GetType() == expected); + } + + [Theory] + [InlineData("FLOOR(DB)")] + [InlineData("SIGN(DB)")] + [InlineData("SQRT(DB)")] + [InlineData("LOG(DB, 2)")] + [InlineData("LOG(2, DB)")] + [InlineData("POWER(DB, 2)")] + [InlineData("POWER(2, DB)")] + [InlineData("ATAN2(DB, 1)")] + [InlineData("ATAN2(1, DB)")] + [InlineData("DEGREES(DB)")] + public void Null_gives_null(string expression) => Assert.Null(Scalar(expression, id: 2)); + + [Theory] + [InlineData("SQRT(-1)")] + [InlineData("LN(0)")] + [InlineData("LOG10(-1)")] + [InlineData("LOG(1, 5)")] + [InlineData("LOG(0, 5)")] + [InlineData("LOG(2, 0)")] + [InlineData("ASIN(2)")] + [InlineData("ACOS(-1.5)")] + [InlineData("POWER(-8, 0.5)")] + public void An_argument_outside_the_domain_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("SINH(1000)")] + [InlineData("POWER(10, 400)")] + public void A_result_past_a_double_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("SQRT()")] + [InlineData("LOG(1, 2, 3)")] + [InlineData("POWER(2)")] + [InlineData("ATAN2(1)")] + [InlineData("PI(1)")] + public void The_wrong_number_of_arguments_is_refused(string expression) => + Assert.Throws(() => Scalar(expression)); +} diff --git a/test/LibRed.Engine.Tests/StrCompStrConvTests.cs b/test/LibRed.Engine.Tests/StrCompStrConvTests.cs index 56790d41c..2cbe59fa0 100644 --- a/test/LibRed.Engine.Tests/StrCompStrConvTests.cs +++ b/test/LibRed.Engine.Tests/StrCompStrConvTests.cs @@ -50,5 +50,5 @@ public void StrConv_vbFromUnicode_128_combines_char_pairs() [Fact] public void StrConv_mode_4_is_rejected() // narrow->wide errors in the JES - => Assert.Throws(() => Eval("StrConv('hello', 4)")); + => Assert.Throws(() => Eval("StrConv('hello', 4)")); } diff --git a/test/LibRed.Engine.Tests/StringFunctionTests.cs b/test/LibRed.Engine.Tests/StringFunctionTests.cs index 502e07eb5..b080ad6b4 100644 --- a/test/LibRed.Engine.Tests/StringFunctionTests.cs +++ b/test/LibRed.Engine.Tests/StringFunctionTests.cs @@ -1,96 +1,232 @@ -using LibRed; +using System.Globalization; using LibRed.Engine; using Xunit; namespace LibRed.Engine.Tests; -public class StringFunctionTests +/// +/// The string functions: how each reads a value that is not text, reads its positions and counts, compares text, +/// and — for Asc, Chr and their W and B forms — maps characters. The expected values were measured against ACE, +/// except that a Null argument gives Null where ACE raises an error. +/// +public class StringFunctionTests(StringFunctionTests.Database database) + : TempDatabaseTest, IClassFixture { - private static string Fresh() - { - string path = TemporaryDatabase.CopyPath(Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "strfn-"); - return path; - } - - private static object? Scalar(string expr) - { - string path = Fresh(); - try - { - using var db = JetDatabase.Open(path, readOnly: false); - var e = new QueryEngine(db); - e.ExecuteNonQuery("CREATE TABLE One (Id LONG)"); - e.ExecuteNonQuery("INSERT INTO One (Id) VALUES (1)"); - return e.ExecuteQuery($"SELECT {expr} FROM One").Rows.First()[0]; - } - finally { TemporaryDatabase.Delete(path); } - } - - [Fact] - public void Length_case_and_trim() - { - Assert.Equal(5, Scalar("LEN('hello')")); - Assert.Equal("hello", Scalar("LCASE('HeLLo')")); - Assert.Equal("HELLO", Scalar("UCASE('HeLLo')")); - Assert.Equal("hi", Scalar("TRIM(' hi ')")); - Assert.Equal("hi ", Scalar("LTRIM(' hi ')")); - Assert.Equal(" hi", Scalar("RTRIM(' hi ')")); - } - - [Fact] - public void Left_right_mid_are_one_based() - { - Assert.Equal("he", Scalar("LEFT('hello', 2)")); - Assert.Equal("hi", Scalar("LEFT('hi', 5)")); // n > length → whole - Assert.Equal("", Scalar("LEFT('hi', 0)")); - Assert.Equal("lo", Scalar("RIGHT('hello', 2)")); - Assert.Equal("ello", Scalar("MID('hello', 2)")); // 1-based, to end - Assert.Equal("ell", Scalar("MID('hello', 2, 3)")); - Assert.Equal("", Scalar("MID('hello', 10)")); // start past end - } - - [Fact] - public void Instr_is_one_based_with_optional_start_and_compare() - { - Assert.Equal(3, Scalar("INSTR('hello', 'l')")); - Assert.Equal(0, Scalar("INSTR('hello', 'z')")); - Assert.Equal(4, Scalar("INSTR(4, 'hello', 'l')")); // optional leading start - Assert.Equal(1, Scalar("INSTR('Hello', 'h')")); // case-insensitive by default - Assert.Equal(0, Scalar("INSTR(1, 'Hello', 'h', 0)")); // compare 0 = binary/case-sensitive - } - - [Fact] - public void Replace_with_optional_start_and_count() - { - Assert.Equal("aXcaXc", Scalar("REPLACE('abcabc', 'b', 'X')")); - Assert.Equal("Jello", Scalar("REPLACE('Hello', 'h', 'J')")); // case-insensitive - Assert.Equal("bba", Scalar("REPLACE('aaa', 'a', 'b', 1, 2)")); // at most 2 - } - - [Fact] - public void String_functions_propagate_null() - { - Assert.Null(Scalar("LEN(NULL)")); - Assert.Null(Scalar("LEFT(NULL, 2)")); - Assert.Null(Scalar("MID(NULL, 1)")); - Assert.Null(Scalar("INSTR('a', NULL)")); - // Note: REPLACE does NOT propagate null — ACE raises "Data type mismatch" (see InstrRevEdgeCasesTests / - // MidReplaceEdgeCasesTests for the functions that error on null instead of propagating). - } - - // The Left()/Right() functions coexist with LEFT/RIGHT JOIN in the same query. - [Fact] - public void Left_function_and_left_join_coexist() - { - string path = Fresh(); - try - { - using var db = JetDatabase.Open(path); - int rows = new QueryEngine(db).ExecuteQuery( - "SELECT LEFT(c.CompanyName, 3) FROM Customers AS c " + - "LEFT JOIN Orders AS o ON c.CustomerID = o.CustomerID").Rows.Count(); - Assert.True(rows > 0); - } - finally { TemporaryDatabase.Delete(path); } - } + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, DC DECIMAL(18,4), NT TEXT(60), DT DATETIME, G GUID, B BINARY(4))", + "INSERT INTO T (Id, DC, DT) VALUES (1, 4.5, #2020-01-02#)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + "UPDATE T SET B = 0x41004200", + ]; + + public sealed class Database() : SharedDatabase("string-functions-", Setup); + + // The code page and the regional formats come from the culture, so each query runs under en-US. + private object? Scalar(string expression) => database.Scalar($"SELECT {expression} FROM T", EnUs); + + [Theory] + [InlineData("UCASE(TRUE)", "-1")] + [InlineData("MID(TRUE, 2)", "1")] + [InlineData("STRREVERSE(TRUE)", "1-")] + [InlineData("TRIM(DT)", "1/2/2020")] + [InlineData("RIGHT(DT, 2)", "20")] + [InlineData("TRIM(DC)", "4.5")] + [InlineData("RIGHT(DC, 2)", ".5")] + [InlineData("UCASE(CSNG(1/3))", "0.3333333")] + [InlineData("TRIM(1/3)", "0.333333333333333")] + [InlineData("LEFT(G, 5)", "{0011")] + [InlineData("LCASE(G)", "{00112233-4455-6677-8899-aabbccddeeff}")] + [InlineData("RIGHT(G, 2)", "F}")] + [InlineData("LEFT(B, 2)", "AB")] + [InlineData("MID(B, 2)", "B")] + [InlineData("STRREVERSE(B)", "BA")] + [InlineData("REPLACE(TRUE, '1', '2')", "-2")] + public void A_value_that_is_not_text_is_read_as_cstr_writes_it(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("LEN(TRUE)", 2)] + [InlineData("LEN(DT)", 8)] + [InlineData("LEN(DC)", 3)] + [InlineData("LEN(G)", 38)] + [InlineData("LEN(B)", 2)] + [InlineData("LENB(TRUE)", 4)] + [InlineData("LENB(G)", 76)] + [InlineData("ASC(TRUE)", 45)] + [InlineData("ASC(G)", 123)] + [InlineData("ASC(B)", 65)] + [InlineData("INSTR(TRUE, '1')", 2)] + [InlineData("INSTR(G, '1')", 4)] + [InlineData("STRCOMP(TRUE, '-1')", 0)] + [InlineData("STRCOMP(B, 'abc')", -1)] + public void Lengths_and_positions_count_the_cstr_text(string expression, int expected) => + Assert.Equal(expected, Convert.ToInt32(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("CHR(65)", "A")] + [InlineData("CHR(128)", "€")] + [InlineData("CHR(130)", "‚")] + [InlineData("CHR(150)", "–")] + [InlineData("CHR(159)", "Ÿ")] + [InlineData("CHR(255)", "ÿ")] + [InlineData("CHRW(8364)", "€")] + [InlineData("CHRW(233)", "é")] + [InlineData("CHRW(65535)", "￿")] + [InlineData("CHRW(-1)", "￿")] + [InlineData("CHRW(-32768)", "耀")] + [InlineData("STRING(3, 321)", "AAA")] + [InlineData("STRING(3, -1)", "ÿÿÿ")] + [InlineData("STRING(3, TRUE)", "ÿÿÿ")] + [InlineData("STRING(3, 8364)", "¬¬¬")] + [InlineData("STRING(3, 'xyz')", "xxx")] + public void Chr_uses_the_ansi_code_page_and_chrw_utf16(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("ASC('€')", 128)] + [InlineData("ASC('Ā')", 65)] + [InlineData("ASC('İ')", 73)] + [InlineData("ASC('あ')", 63)] + [InlineData("ASC('fi')", 63)] + [InlineData("ASC(CHRW(8364))", 128)] + [InlineData("ASC(CHRW(128))", 63)] + [InlineData("ASCW(CHR(128))", 8364)] + [InlineData("ASCW(CHR(159))", 376)] + [InlineData("ASCW('fi')", -1279)] + [InlineData("ASCW(CHRW(-1))", -1)] + [InlineData("ASCB('€')", 172)] + public void Asc_gives_the_ansi_code_and_ascw_the_signed_utf16_unit(string expression, int expected) => + Assert.Equal(expected, Convert.ToInt32(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("CHR(256)")] + [InlineData("CHR(-1)")] + [InlineData("CHR(TRUE)")] + [InlineData("CHR(8364)")] + [InlineData("CHRW(65536)")] + [InlineData("CHRW(-32769)")] + [InlineData("ASC('')")] + [InlineData("ASCW('')")] + [InlineData("STRING(3, '')")] + [InlineData("STRING(-1, 'x')")] + [InlineData("STRING(TRUE, 'x')")] + [InlineData("SPACE(-1)")] + [InlineData("LEFT('abcdef', TRUE)")] + [InlineData("MID('abcdef', TRUE)")] + [InlineData("MID('abc', 0)")] + [InlineData("INSTR(0, 'abcabc', 'b')")] + [InlineData("INSTR(-1, 'abcabc', 'b')")] + [InlineData("REPLACE('abcabc', 'b', 'X', TRUE)")] + [InlineData("MIDB('abc', 0)")] + [InlineData("MIDB('abc', -1)")] + [InlineData("MIDB('abc', 1, -1)")] + [InlineData("LEFTB('abc', -1)")] + [InlineData("RIGHTB('abc', -1)")] + [InlineData("INSTRB(0, 'abc', 'b')")] + [InlineData("INSTRB(-1, 'abc', 'b')")] + [InlineData("INSTR(1, 'aBc', 'b', 2)")] + [InlineData("INSTR(1, 'aBc', 'b', -1)")] + [InlineData("INSTR(1, 'aBc', 'b', 1.5)")] + [InlineData("INSTR(1, 'aBc', 'b', 20000)")] + [InlineData("STRCOMP('a', 'A', 2)")] + [InlineData("STRCOMP('a', 'A', TRUE)")] + public void An_argument_out_of_range_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("LEFT('abcdef', '$2')", "ab")] + [InlineData("LEFT('abcdef', 2.5)", "ab")] + [InlineData("LEFT('abcdef', 3.5)", "abcd")] + [InlineData("LEFT('abcdef', #1899-12-31#)", "a")] + [InlineData("MID('abcdef', 2, '$2')", "bc")] + [InlineData("SPACE('$2') & '|'", " |")] + [InlineData("STRING('$2', 'x')", "xx")] + [InlineData("REPLACE('abcabc', 'b', 'X', '$2')", "XcaXc")] + [InlineData("REPLACE('abcabc', 'b', 'X', 1, TRUE)", "aXcaXc")] + [InlineData("REPLACE('abcabc', 'b', 'X', 1, #1899-12-31#)", "aXcabc")] + [InlineData("LEFTB('abc', '$2')", "a")] + [InlineData("MIDB('abc', '$3')", "bc")] + [InlineData("RIGHTB('abc', 2.5)", "c")] + [InlineData("RIGHTB('abc', 0)", "")] + public void Positions_and_counts_are_read_as_numbers(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("INSTR('$2', 'abcabc', 'b')", 2)] + [InlineData("INSTR(#1899-12-31#, 'abcabc', 'b')", 2)] + [InlineData("INSTRREV('abcabc', 'b', '$2')", 2)] + [InlineData("INSTRREV('abcabc', 'b', TRUE)", 5)] + [InlineData("INSTRREV('abcabc', 'b', 7)", 0)] + [InlineData("INSTR(5, 'abc', '')", 5)] + [InlineData("INSTR(4, 'abc', '')", 4)] + [InlineData("INSTR('', '')", 0)] + [InlineData("INSTR(1, 'aBc', 'b', 0)", 0)] + [InlineData("INSTR(1, 'aBc', 'b', 1)", 2)] + [InlineData("INSTR(1, 'aBc', 'b', 3.5)", 2)] + [InlineData("INSTR(1, 'aBc', 'b', 1033)", 2)] + [InlineData("INSTR(1, 'aBc', 'b', #1899-12-31#)", 2)] + [InlineData("INSTRB('$2', 'abc', 'b')", 3)] + [InlineData("INSTRB(7, 'abc', 'b')", 0)] + [InlineData("INSTR(1, 'ßx', 'SS', 1033)", 1)] + [InlineData("INSTR(1, 'ßx', 'SS', 0)", 0)] + [InlineData("STRCOMP('a', 'A', 1031)", 0)] + public void Positions_and_compare_modes_are_read_as_numbers(string expression, int expected) => + Assert.Equal(expected, Convert.ToInt32(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("INSTR('straße', 'SS')", 5)] + [InlineData("INSTR('STRASSE', 'ß')", 5)] + [InlineData("INSTR('ÆØ', 'ae')", 1)] + [InlineData("INSTR('Éclair', 'e')", 0)] + [InlineData("STRCOMP('ß', 'ss')", 0)] + [InlineData("STRCOMP('Æ', 'AE')", 0)] + [InlineData("STRCOMP('a-b', 'ab')", 1)] + [InlineData("STRCOMP('ab', 'a-b')", -1)] + [InlineData("STRCOMP('co-op', 'coop')", 1)] + [InlineData("STRCOMP('ß', 'ss', 1033)", 0)] + [InlineData("STRCOMP('é', 'E')", 1)] + [InlineData("STRCOMP('é', 'É')", 0)] + [InlineData("STRCOMP('a', 'a ')", -1)] + [InlineData("STRCOMP('a', 'B', 0)", 1)] + public void Text_compares_in_the_database_sort_order(string expression, int expected) => + Assert.Equal(expected, Convert.ToInt32(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("'ß' = 'ss'", true)] + [InlineData("'a-b' > 'ab'", true)] + [InlineData("'a-b' < 'ac'", true)] + [InlineData("'é' < 'f'", true)] + [InlineData("'café' = 'cafe'", false)] + public void Comparison_operators_use_the_same_order(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("REPLACE('straße', 'SS', '-')", "stra-e")] + [InlineData("REPLACE('aXbxc', 'x', '-')", "a-b-c")] + [InlineData("REPLACE('aXbxc', 'x', '-', 1, -1, 0)", "aXb-c")] + [InlineData("REPLACE('abcabc', 'b', 'X', 3)", "caXc")] + [InlineData("REPLACE('abcabc', 'b', 'X', 7)", "")] + [InlineData("REPLACE('aBc', 'b', 'X', 1, -1, 3)", "aXc")] + public void Replace_finds_text_in_the_database_sort_order(string expression, string expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("MID('abcdef', 2, NULL)")] + [InlineData("INSTR(NULL, 'abcabc', 'b')")] + [InlineData("INSTR(1, 'aBc', 'b', NULL)")] + [InlineData("STRCOMP('a', 'A', NULL)")] + [InlineData("SPACE(NULL)")] + [InlineData("STRING(NULL, 'x')")] + [InlineData("STRING(3, NULL)")] + [InlineData("CHR(NULL)")] + [InlineData("REPLACE('abcabc', 'b', 'X', 1, NULL)")] + [InlineData("ASC(NT)")] + [InlineData("STRREVERSE(NULL)")] + [InlineData("LEFTB('abc', NULL)")] + [InlineData("INSTRB(NULL, 'abc', 'b')")] + public void Null_gives_null(string expression) => + Assert.Null(Scalar(expression)); } diff --git a/test/LibRed.Engine.Tests/TimeSpanParameterTests.cs b/test/LibRed.Engine.Tests/TimeSpanParameterTests.cs new file mode 100644 index 000000000..e0e9d879f --- /dev/null +++ b/test/LibRed.Engine.Tests/TimeSpanParameterTests.cs @@ -0,0 +1,130 @@ +using LibRed; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +// A TimeSpan or TimeOnly parameter is the time Jet stores — the time on the 1899-12-30 epoch — wherever it is read: +// saved, compared, passed to a function. Beside a date in + or - it is a span instead, so a date less an hour is a +// date, not the day count a date less a date is. A time written into the SQL keeps Access's reading. +public class TimeSpanParameterTests : TempDatabaseTest +{ + private static QueryEngine Fresh() + { + string path = TemporaryDatabase.CopyPath( + Path.Combine(AppContext.BaseDirectory, "Data", "Northwind.accdb"), "timespan-param-"); + var engine = new QueryEngine(TemporaryDatabase.OpenTracked(path, readOnly: false)); + engine.ExecuteNonQuery("CREATE TABLE Q (K LONG, A DATETIME, N DATETIME, B DATETIME2)"); + engine.ExecuteNonQuery("INSERT INTO Q (K, A, N, B) VALUES (1, #2020-02-29 17:55:00#, NULL, #2020-02-29 17:55:00#)"); + return engine; + } + + private static readonly DateTime At = new(2020, 2, 29, 17, 55, 0); + + private static Dictionary Bind(object value) => new() { ["ts"] = value }; + + [Theory] + [InlineData("A + @ts", 1)] + [InlineData("@ts + A", 1)] + [InlineData("A - @ts", -1)] + [InlineData("B + @ts", 1)] + [InlineData("B - @ts", -1)] + public void A_date_moves_by_the_span(string expression, int hours) + { + foreach (object span in new object[] { TimeSpan.FromHours(1), new TimeOnly(1, 0) }) + { + var result = Fresh().ExecuteQuery($"SELECT {expression} FROM Q", Bind(span)); + Assert.Equal(typeof(DateTime), result.ColumnTypes[0]); + Assert.Equal(At.AddHours(hours), result.Rows.Single()[0]); + } + } + + // A span below zero moves the date back. As a date it would be before the epoch, where the time counts away from + // zero, and the sum would land 1.75 days off rather than a quarter of one. + [Theory] + [InlineData("A + @ts", -6)] + [InlineData("A - @ts", 6)] + public void A_negative_span_moves_the_other_way(string expression, int hours) => + Assert.Equal(At.AddHours(hours), Fresh().ExecuteQuery($"SELECT {expression} FROM Q", Bind(TimeSpan.FromHours(-6))).Rows.Single()[0]); + + // A span stays one through negation, sums and differences of spans, scaling by a number and a choice among spans, + // so the date moves by the whole of it however the SQL groups it. @a is an hour, @b half of one. + [Theory] + [InlineData("A - (@a + @b)", -90)] + [InlineData("A + (@a - @b)", 30)] + [InlineData("A + (@b - @a)", -30)] // a negative total moves it back + [InlineData("A - (-@a)", 60)] + [InlineData("A + @a * 2", 120)] + [InlineData("A + 2 * @a", 120)] + [InlineData("A + @a / 2", 30)] + [InlineData("A + @a * K", 60)] // K is 1: a factor from the row + [InlineData("A + IIF(K = 1, @a, @b)", 60)] + [InlineData("A + IIF(K = 2, @a, @b)", 30)] + [InlineData("A + COALESCE(NULL, @b)", 30)] + [InlineData("A - CASE WHEN K = 1 THEN @a ELSE @b END", -60)] + [InlineData("B - (@a + @b)", -90)] + [InlineData("(@a + @b) + A", 90)] + public void A_date_moves_by_a_span_however_it_is_built(string expression, int minutes) + { + var result = Fresh().ExecuteQuery($"SELECT {expression} FROM Q", + new Dictionary { ["a"] = TimeSpan.FromHours(1), ["b"] = TimeSpan.FromMinutes(30) }); + Assert.Equal(typeof(DateTime), result.ColumnTypes[0]); + Assert.Equal(At.AddMinutes(minutes), result.Rows.Single()[0]); + } + + [Fact] + public void A_choice_of_a_null_moves_nothing_and_is_null() => + Assert.Null(Fresh().ExecuteQuery("SELECT A + IIF(K = 2, @a, NULL) FROM Q", + new Dictionary { ["a"] = TimeSpan.FromHours(1) }).Rows.Single()[0]); + + // A span with a number added is no span — the number is a count of days, and the sum is a date — so the date + // less it is a day count, as Access has it. + [Fact] + public void A_span_plus_a_number_is_a_date_not_a_span() => + Assert.IsType(Fresh().ExecuteQuery("SELECT A - (@a + 1) FROM Q", + new Dictionary { ["a"] = TimeSpan.FromHours(1) }).Rows.Single()[0]); + + [Fact] + public void A_span_divided_by_zero_is_a_division_by_zero() => + Assert.Throws(() => Fresh().ExecuteQuery("SELECT A + @a / 0 FROM Q", + new Dictionary { ["a"] = TimeSpan.FromHours(1) }).Rows.ToList()); + + [Fact] + public void A_null_date_stays_null_and_is_still_declared_a_date() + { + var result = Fresh().ExecuteQuery("SELECT N - @ts FROM Q", Bind(TimeSpan.FromHours(1))); + Assert.Equal(typeof(DateTime), result.ColumnTypes[0]); + Assert.Null(result.Rows.Single()[0]); + } + + [Fact] + public void A_time_written_into_the_sql_is_still_a_date() => + Assert.IsType(Fresh().ExecuteQuery("SELECT A - #01:00:00# FROM Q").Rows.Single()[0]); + + [Fact] + public void On_its_own_it_is_the_time_on_the_epoch() + { + var result = Fresh().ExecuteQuery("SELECT @ts, HOUR(@ts) FROM Q", Bind(new TimeSpan(9, 30, 0))); + Assert.Equal(typeof(DateTime), result.ColumnTypes[0]); + Assert.Equal(new object?[] { new DateTime(1899, 12, 30, 9, 30, 0), 9 }, result.Rows.Single()); + } + + // Saved, it is stored as Jet stores a time — in a Date/Time column and a Date/Time Extended one alike — and a + // comparison against it finds the row. + [Fact] + public void Saved_it_is_the_time_on_the_epoch() + { + QueryEngine engine = Fresh(); + var time = new TimeSpan(0, 13, 45, 30, 250); + engine.ExecuteNonQuery("CREATE TABLE T (K LONG, T DATETIME, T2 DATETIME2)"); + engine.ExecuteNonQuery("INSERT INTO T (K, T, T2) VALUES (1, @ts, @ts)", Bind(time)); + engine.ExecuteNonQuery("INSERT INTO T (K, T, T2) VALUES (2, @ts, @ts)", Bind(new TimeOnly(8, 15))); + + var epoch = new DateTime(1899, 12, 30); + var rows = engine.ExecuteQuery("SELECT T, T2 FROM T ORDER BY K").Rows.ToList(); + Assert.Equal(new object?[] { epoch + time, epoch + time }, rows[0]); + Assert.Equal(new object?[] { epoch.AddHours(8.25), epoch.AddHours(8.25) }, rows[1]); + + Assert.Equal(1, engine.ExecuteQuery("SELECT K FROM T WHERE T = @ts AND T2 = @ts", Bind(time)).Rows.Single()[0]); + } +} diff --git a/test/LibRed.Engine.Tests/TrimFunctionsTests.cs b/test/LibRed.Engine.Tests/TrimFunctionsTests.cs index c34e6474e..da210b6ac 100644 --- a/test/LibRed.Engine.Tests/TrimFunctionsTests.cs +++ b/test/LibRed.Engine.Tests/TrimFunctionsTests.cs @@ -4,8 +4,8 @@ namespace LibRed.Engine.Tests; -// Trim/LTrim/RTrim are single-argument and remove ONLY spaces (not tabs or other whitespace, and no trim-char -// parameter) — verified vs ACE. NULL-propagating. +// Trim/LTrim/RTrim are single-argument and remove ONLY spaces — the space and the ideographic space U+3000, in any +// mixture; not tabs or other whitespace, and no trim-char parameter — verified vs ACE. NULL-propagating. public class TrimFunctionsTests : TempDatabaseTest { private static QueryEngine Fresh() @@ -31,6 +31,23 @@ public void Trim_removes_only_spaces_not_tabs() // a tab (Chr(9)) either side is preserved — ACE Trim removes spaces only. => Assert.Equal("\thi\t", Convert.ToString(Eval("Trim(Chr(9) & 'hi' & Chr(9))"))); + // A space, U+3000, a space, 'a', U+3000, a space, U+3000: ACE strips the whole run either side. + [Theory] + [InlineData("Trim", "a")] + [InlineData("LTrim", "a　 　")] + [InlineData("RTrim", " 　 a")] + public void The_ideographic_space_is_trimmed_as_a_space(string function, string expected) + => Assert.Equal(expected, Convert.ToString(Eval( + $"{function}(' ' & ChrW(12288) & ' ' & 'a' & ChrW(12288) & ' ' & ChrW(12288))"))); + + // No other space is: a no-break space, an en space, a zero-width space. + [Theory] + [InlineData(160)] + [InlineData(8194)] + [InlineData(8203)] + public void Other_unicode_spaces_are_kept(int codePoint) + => Assert.Equal(3, Convert.ToString(Eval($"Trim(ChrW({codePoint}) & 'a' & ChrW({codePoint}))"))!.Length); + [Theory] [InlineData("Trim(Null)")] [InlineData("LTrim(Null)")] diff --git a/test/LibRed.Engine.Tests/UnaryAndBitwiseOperatorTests.cs b/test/LibRed.Engine.Tests/UnaryAndBitwiseOperatorTests.cs new file mode 100644 index 000000000..80b0dd389 --- /dev/null +++ b/test/LibRed.Engine.Tests/UnaryAndBitwiseOperatorTests.cs @@ -0,0 +1,321 @@ +using System.Globalization; +using LibRed.Engine; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Unary minus and plus, and the bitwise operators BNOT BAND BOR BXOR: how each reads its +/// operand, the widths the bits are combined at, and where the operators bind. The expected values were measured +/// against ACE, except where ACE reads the raw bytes of a value that is not a whole number and returns garbage or +/// crashes; LibRed reads such a value as a number there. +/// +public class UnaryAndBitwiseOperatorTests(UnaryAndBitwiseOperatorTests.Database database) + : TempDatabaseTest, IClassFixture +{ + private static readonly CultureInfo EnUs = CultureInfo.GetCultureInfo("en-US"); + + private static readonly string[] Setup = + [ + "CREATE TABLE T (Id LONG, BT BYTE, SI SHORT, LG LONG, DC DECIMAL(18,4), TN TEXT(60), MM MEMO, NT TEXT(60), " + + "DT DATETIME, YN YESNO, G GUID)", + "INSERT INTO T (Id, BT, SI, LG, DC, TN, MM, DT, YN) VALUES (1, 1, 2, 3, 4.5, '7', '8', #2020-01-02 12:00:00#, TRUE)", + "UPDATE T SET G = {00112233-4455-6677-8899-AABBCCDDEEFF}", + ]; + + public sealed class Database() : SharedDatabase("unary-bitwise-", Setup); + + // Text is read as a number in the regional separators, so each query runs under en-US whatever the machine's + // culture. + private object? Query(string sql) => database.Scalar(sql, EnUs); + + private object? Scalar(string expression) => Query($"SELECT {expression} FROM T"); + + [Theory] + [InlineData("1 BOR 40000", 40001)] + [InlineData("12 BAND 10", 8)] + [InlineData("12 BXOR 10", 6)] + [InlineData("BT BOR SI", 3)] + [InlineData("40000 BAND BT", 0)] + [InlineData("-1 BAND 3", 3)] + [InlineData("TRUE BAND 1", 1)] + [InlineData("TRUE BOR BT", -1)] + // Two 16-bit operands give an Integer. + [InlineData("CINT(-2) BAND CINT(-3)", (short)-4)] + [InlineData("TRUE BXOR SI", (short)-3)] + [InlineData("TRUE BAND TRUE", (short)-1)] + [InlineData("SI BOR SI", (short)2)] + [InlineData("YN BAND YN", (short)-1)] + [InlineData("YN BXOR SI", (short)-3)] + [InlineData("BT BAND BT", 1)] + public void Whole_numbers_combine_their_bits(string expression, object expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("1 BOR TRUE", 65535)] + [InlineData("BT BOR TRUE", 65535)] + [InlineData("40000 BXOR TRUE", 25535)] + [InlineData("LG BXOR TRUE", 65532)] + [InlineData("1 BOR CINT(-2)", 65535)] + [InlineData("CLNG(-1) BAND CINT(1)", -65535)] + [InlineData("70000 BAND CINT(1)", 65536)] + [InlineData("70000 BOR TRUE", 131071)] + [InlineData("70000 BXOR TRUE", 126607)] + [InlineData("CLNG(-40000) BAND TRUE", -40000)] + [InlineData("CLNG(-40000) BAND FALSE", -65536)] + [InlineData("CLNG(70000) BOR CINT(-32768)", 102768)] + public void A_sixteen_bit_right_operand_changes_only_the_low_sixteen_bits(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("TRUE BAND 40000", -25536)] + [InlineData("SI BOR 40000", -25534)] + [InlineData("CINT(1) BOR 70000", 4465)] + [InlineData("FALSE BOR 70000", 4464)] + [InlineData("TRUE BAND -40000", 25536)] + [InlineData("CINT(-2) BOR 1", -1)] + public void A_sixteen_bit_left_operand_gives_a_sixteen_bit_result(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("'2.5' BOR 1", 3)] + [InlineData("'2.5' BAND '2.5'", 2)] + [InlineData("'1' BAND '2.5'", 0)] + [InlineData("'2.5' BOR DC", 6)] + [InlineData("TN BAND 1", 1)] + [InlineData("1.5 BAND '2.5'", 2)] + [InlineData("DT BAND 40000", 34816)] + [InlineData("1 BOR DT", 43833)] + [InlineData("1.5 BOR DT", 43834)] + [InlineData("#2020-01-02# BAND MM", 8)] + [InlineData("#2020-01-02# BOR #2020-01-02#", 43832)] + public void Other_values_are_read_as_whole_numbers_rounding_half_to_even(string expression, int expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("NULL BAND 0")] + [InlineData("0 BOR NULL")] + [InlineData("NULL BXOR NULL")] + [InlineData("'abc' BAND NULL")] + [InlineData("NT BAND 1")] + [InlineData("BNOT NULL")] + [InlineData("BNOT NT")] + public void Null_gives_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("'abc' BAND 1")] + [InlineData("1 BOR ''")] + [InlineData("G BAND 1")] + [InlineData("BNOT 'abc'")] + [InlineData("BNOT G")] + public void A_value_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("2147483647.5 BAND 1")] + [InlineData("'2147483648' BAND 1")] + [InlineData("BNOT 1E10")] + [InlineData("BNOT 2147483647.5")] + [InlineData("BNOT '-2147483649'")] + public void A_whole_number_past_a_long_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("BNOT 5", -6)] + [InlineData("BNOT TRUE", (short)0)] + [InlineData("BNOT FALSE", (short)-1)] + [InlineData("BNOT SI", (short)-3)] + [InlineData("BNOT YN", (short)0)] + [InlineData("BNOT BNOT YN", (short)-1)] + [InlineData("BNOT CINT(-32768)", (short)32767)] + [InlineData("BNOT BT", -2)] + [InlineData("BNOT CLNG(-2147483648)", 2147483647)] + [InlineData("BNOT 1.5", -3)] + [InlineData("BNOT 2.5", -3)] + [InlineData("BNOT -0.5", -1)] + [InlineData("BNOT -1.5", 1)] + [InlineData("BNOT -2147483648.5", 2147483647)] + [InlineData("BNOT 2147483647.4", -2147483648)] + [InlineData("BNOT '7'", -8)] + [InlineData("BNOT '2.5'", -3)] + [InlineData("BNOT #2020-01-02#", -43833)] + [InlineData("BNOT DT", -43833)] + [InlineData("BNOT BNOT 5", 5)] + [InlineData("-BNOT 5", 6)] + [InlineData("BNOT -5", 4)] + public void Bnot_flips_every_bit(string expression, object expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("BNOT 1 + 1", -3)] + [InlineData("BNOT 2 * 3", -7)] + [InlineData("BNOT 2 ^ 2", -5)] + [InlineData("BNOT 3 = 3 + 10", (short)-1)] + [InlineData("BNOT 0 IS NULL", (short)-1)] + [InlineData("BNOT NULL IS NULL", (short)0)] + [InlineData("BNOT 0 BAND 3", 3)] + [InlineData("NOT 0 BAND 1", 1)] + [InlineData("NOT 1 BAND 2", 0)] + [InlineData("2 AND 1 BAND 3", 3)] + [InlineData("0 OR 0 BOR 4", 4)] + [InlineData("0 XOR 0 BXOR 4", 4)] + [InlineData("4 BOR 0 AND 0", 4)] + [InlineData("0 AND 0 BOR 4", 4)] + [InlineData("4 BXOR 0 OR 0", 4)] + [InlineData("0 OR 0 BXOR 4", 4)] + [InlineData("5 BOR 3 BAND 8", 5)] + [InlineData("5 BXOR 3 BAND 1", 4)] + [InlineData("5 BXOR 3 BOR 1", 6)] + [InlineData("5 BAND 3 BOR 8", 9)] + [InlineData("1 BAND 3 = 1", 0)] + [InlineData("3 BAND 1 = 1", 3)] + [InlineData("12 BAND 10 + 1", 8)] + [InlineData("1 BAND 3 IS NULL", 0)] + public void Bnot_sits_with_not_and_each_bitwise_operator_with_its_logical_one(string expression, object expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("3 BAND 1 AND 2", true)] + [InlineData("4 BOR 0 OR 0", true)] + [InlineData("4 BXOR 0 XOR 0", true)] + [InlineData("4 EQV 4 BXOR 4", false)] + [InlineData("0 BXOR 4 EQV 4", true)] + public void A_logical_operator_applied_last_gives_a_boolean(string expression, bool expected) => + Assert.Equal(expected, Scalar(expression)); + + [Theory] + [InlineData("-'2.5'", -2.5)] + [InlineData("-'1e2'", -100)] + [InlineData("-'&H10'", -16)] + [InlineData("-'1,000'", -1000)] + [InlineData("-'($5)'", 5)] + [InlineData("-' 1 '", -1)] + [InlineData("-TN", -7)] + [InlineData("-'2147483648'", -2147483648)] + [InlineData("-TRUE", 1)] + [InlineData("-FALSE", 0)] + [InlineData("-SI", -2)] + [InlineData("-BT", -1)] + [InlineData("-DC", -4.5)] + [InlineData("-CINT(-32767)", 32767)] + [InlineData("-CLNG(-2147483647)", 2147483647)] + [InlineData("-CBYTE(255)", -255)] + [InlineData("-CCUR(1)", -1)] + [InlineData("-(-32767 - 1)", 32768)] + [InlineData("-SI - 32767", -32769)] + [InlineData("-(SI * 16384)", -32768)] + [InlineData("--2", 2)] + [InlineData("---2", -2)] + [InlineData("--2 ^ 2", -4)] + [InlineData("--.5", 0.5)] + [InlineData("1--2", 3)] + [InlineData("- - -2", -2)] + [InlineData("-(-(-2))", -2)] + public void Minus_negates_the_value_read_as_a_number(string expression, double expected) => + Assert.Equal(expected, Convert.ToDouble(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("-#2020-01-02#", "1779-12-27 00:00:00")] + [InlineData("-DT", "1779-12-27 12:00:00")] + [InlineData("-#1899-12-30 12:00#", "1899-12-30 12:00:00")] + public void Minus_negates_a_dates_serial_and_keeps_a_date(string expression, string expected) => + Assert.Equal(expected, Assert.IsType(Scalar(expression)).ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("-CINT(-32768)")] + [InlineData("-CLNG(-2147483648)")] + public void Minus_past_the_operands_range_is_an_overflow(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("-'abc'")] + [InlineData("-''")] + [InlineData("-G")] + public void Minus_on_a_value_that_is_not_a_number_is_a_type_mismatch(string expression) => + Assert.Throws(() => Scalar(expression)); + + [Theory] + [InlineData("-NULL")] + [InlineData("-NT")] + [InlineData("+NULL")] + public void Minus_and_plus_of_null_are_null(string expression) => + Assert.Null(Scalar(expression)); + + [Theory] + [InlineData("-2 ^ 2", 4)] + [InlineData("-2 ^ 3", -8)] + [InlineData("-2.5 ^ 2", 6.25)] + [InlineData("-2E0 ^ 2", 4)] + [InlineData("-0.5 ^ 2", 0.25)] + [InlineData("-2 ^ 2 ^ 2", 16)] + [InlineData("2 ^ -2 ^ 2", 0.0625)] + [InlineData("2 * -2 ^ 2", 8)] + [InlineData("-2 ^ -2", 0.25)] + [InlineData("- 2 ^ 2", -4)] + [InlineData("-(2) ^ 2", -4)] + [InlineData("-.5 ^ 2", -0.25)] + [InlineData("-SI ^ 2", -4)] + [InlineData("-'2' ^ 2", -4)] + [InlineData("-CINT(2) ^ 2", -4)] + [InlineData("- -2 ^ 2", -4)] + [InlineData("-(2 ^ 2)", -4)] + [InlineData("3 - 2 ^ 2", -1)] + [InlineData("1 - -1", 2)] + [InlineData("1 - - 1", 2)] + [InlineData("- 2 - 3", -5)] + [InlineData("-2 MOD 3", -2)] + [InlineData("-7 \\ 2", -3)] + [InlineData("+2 ^ 2", 4)] + [InlineData("+-2 ^ 2", 4)] + public void A_minus_against_a_number_is_part_of_it_otherwise_it_binds_below_power(string expression, double expected) => + Assert.Equal(expected, Convert.ToDouble(Scalar(expression), CultureInfo.InvariantCulture)); + + [Theory] + [InlineData("+1", 1)] + [InlineData("+(1)", 1)] + [InlineData("+1 + +1", 2)] + [InlineData("1 + + 1", 2)] + [InlineData("1 - + 1", 0)] + [InlineData("2 * +3", 6)] + [InlineData("-+1", -1)] + [InlineData("+-1", -1)] + [InlineData("+-+1", -1)] + [InlineData("+SI", 2)] + [InlineData("+'1' + 1", 2)] + public void Plus_leaves_a_number_as_it_is(string expression, double expected) => + Assert.Equal(expected, Convert.ToDouble(Scalar(expression), CultureInfo.InvariantCulture)); + + [Fact] + public void Plus_leaves_text_and_other_values_as_they_are() + { + Assert.Equal("abc", Scalar("+'abc'")); + Assert.Equal("1x", Scalar("+'1' & 'x'")); + Assert.Equal(Guid.Parse("00112233-4455-6677-8899-AABBCCDDEEFF"), Scalar("+G")); + Assert.Equal(new DateTime(2020, 1, 2, 12, 0, 0), Scalar("+DT")); + } + + [Theory] + [InlineData("-- tag\nSELECT 1 FROM T")] + [InlineData("--\tTag\r\nSELECT 1 FROM T")] + [InlineData("--\nSELECT 1 FROM T")] + [InlineData("SELECT 1 FROM T --")] + [InlineData("SELECT 1 FROM T -- trailing")] + [InlineData("--Before\nSELECT 1 FROM T")] + [InlineData("---tag\nSELECT 1 FROM T")] + public void Two_dashes_start_a_comment_unless_a_number_follows(string sql) => + Assert.Equal(1, Query(sql)); + + [Theory] + [InlineData("BNOT 0", 1)] + [InlineData("BNOT -1", 0)] + [InlineData("1 BAND 2", 0)] + [InlineData("1 BAND 3", 1)] + [InlineData("-1", 1)] + [InlineData("-0", 0)] + [InlineData("-'0'", 0)] + [InlineData("-NULL", 0)] + [InlineData("NULL BAND 0", 0)] + public void A_where_condition_uses_the_same_values(string condition, int expected) => + Assert.Equal(expected, Query($"SELECT COUNT(*) FROM T WHERE {condition}")); +} diff --git a/test/LibRed.Engine.Tests/WindowAggregateTests.cs b/test/LibRed.Engine.Tests/WindowAggregateTests.cs new file mode 100644 index 000000000..01a720089 --- /dev/null +++ b/test/LibRed.Engine.Tests/WindowAggregateTests.cs @@ -0,0 +1,135 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Aggregates as window functions, over the standard's default frame: with an ORDER BY, the partition's rows up to +/// the current row and its peers; without one, the whole partition. The values and types are the grouped +/// aggregate's. Access has no window functions; this is a LibRed extension. +/// +public class WindowAggregateTests(WindowAggregateTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Partition 'a' has a tie on V (Ids 2 and 3); partition 'b' has a Null V. T holds an empty text. + private static readonly string[] Setup = + [ + "CREATE TABLE W (Id LONG, G TEXT(10), V LONG, M CURRENCY, T TEXT(10))", + "INSERT INTO W (Id, G, V, M, T) VALUES (1, 'a', 10, 1.5, 'x')", + "INSERT INTO W (Id, G, V, M, T) VALUES (2, 'a', 20, 2.25, 'y')", + "INSERT INTO W (Id, G, V, M, T) VALUES (3, 'a', 20, NULL, '')", + "INSERT INTO W (Id, G, V, M, T) VALUES (4, 'b', 5, 1, 'z')", + "INSERT INTO W (Id, G, V, M, T) VALUES (5, 'b', NULL, 3, 'w')", + ]; + + public sealed class Database() : SharedDatabase("window-aggregate-", Setup); + + private (IReadOnlyList Types, List Rows) Query(string sql) => + database.Query(sql, CultureInfo.InvariantCulture); + + private string ById(string function, string over) => string.Join(" ", + Query($"SELECT Id, {function} OVER ({over}) AS r FROM W ORDER BY Id").Rows + .Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("SUM(V)", "", "1:55 2:55 3:55 4:55 5:55")] + [InlineData("SUM(V)", "PARTITION BY G", "1:50 2:50 3:50 4:5 5:5")] + [InlineData("SUM(V)", "ORDER BY Id", "1:10 2:30 3:50 4:55 5:55")] + [InlineData("SUM(V)", "PARTITION BY G ORDER BY Id DESC", "1:50 2:40 3:20 4:5 5:")] + [InlineData("COUNT(*)", "PARTITION BY G ORDER BY Id", "1:1 2:2 3:3 4:1 5:2")] + [InlineData("COUNT(V)", "PARTITION BY G", "1:3 2:3 3:3 4:1 5:1")] + [InlineData("AVG(V)", "PARTITION BY G", "1:16.666666666666668 2:16.666666666666668 3:16.666666666666668 4:5 5:5")] + [InlineData("MAX(V)", "ORDER BY Id", "1:10 2:20 3:20 4:20 5:20")] + [InlineData("MIN(M)", "PARTITION BY G ORDER BY Id", "1:1.5 2:1.5 3:1.5 4:1 5:1")] + public void An_aggregate_runs_over_the_partition_up_to_the_current_row(string function, string over, string expected) => + Assert.Equal(expected, ById(function, over)); + + // Rows that tie on the window order are peers: the running value includes all of them. Nulls sort first, and + // a sum over nothing but a Null is Null. + [Fact] + public void Peers_share_a_running_value() => + Assert.Equal("1:15 2:55 3:55 4:5 5:", ById("SUM(V)", "ORDER BY V")); + + [Theory] + [InlineData("SUM(V)")] + [InlineData("AVG(V)")] + [InlineData("MIN(T)")] + [InlineData("MAX(T)")] + [InlineData("COUNT(V)")] + [InlineData("COUNT(*)")] + [InlineData("VAR(V)")] + [InlineData("VARP(V)")] + [InlineData("STDEV(M)")] + [InlineData("STDEVP(M)")] + [InlineData("SUM(M)")] + [InlineData("AVG(M)")] + public void Over_the_whole_input_it_is_the_grouped_aggregate(string aggregate) + { + var (groupedTypes, grouped) = Query($"SELECT {aggregate} AS r FROM W"); + var (windowedTypes, windowed) = Query($"SELECT {aggregate} OVER () AS r FROM W"); + + Assert.Equal(groupedTypes[0], windowedTypes[0]); + Assert.All(windowed, row => Assert.Equal(grouped[0][0], row[0])); + } + + [Fact] + public void Over_a_partition_it_is_the_grouped_aggregate_of_that_group() + { + var grouped = Query("SELECT G, SUM(V), STDEVP(M) FROM W GROUP BY G").Rows + .ToDictionary(row => (string)row[0]!, row => (row[1], row[2])); + foreach (object?[] row in Query("SELECT G, SUM(V) OVER (PARTITION BY G), STDEVP(M) OVER (PARTITION BY G) FROM W").Rows) + Assert.Equal(grouped[(string)row[0]!], (row[1], row[2])); + } + + [Theory] + [InlineData("SUM(V) OVER ()", typeof(int))] + [InlineData("SUM(M) OVER ()", typeof(decimal))] + [InlineData("AVG(V) OVER ()", typeof(double))] + [InlineData("COUNT(*) OVER ()", typeof(int))] + [InlineData("MAX(T) OVER ()", typeof(string))] + public void The_column_is_typed_as_the_grouped_aggregate(string expression, Type expected) + { + var (types, rows) = Query($"SELECT {expression} AS r FROM W"); + Assert.Equal(expected, types[0]); + Assert.All(rows, row => Assert.True(row[0] is null || row[0]!.GetType() == expected)); + } + + [Fact] + public void A_windowed_currency_sum_is_a_currency() => + Assert.Equal("Currency", Query("SELECT TYPENAME(SUM(M) OVER ()) AS r FROM W").Rows[0][0]); + + // V runs 10, 20, 20, 5, Null by Id: the second 20 counts once in every frame that holds both. + [Theory] + [InlineData("COUNT(DISTINCT V)", "PARTITION BY G", "1:2 2:2 3:2 4:1 5:1")] + [InlineData("SUM(DISTINCT V)", "ORDER BY Id", "1:10 2:30 3:30 4:35 5:35")] + [InlineData("SUM(DISTINCT V)", "ORDER BY Id ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING", "1:35 2:25 3:25 4:5 5:")] + [InlineData("SUM(DISTINCT V)", "ORDER BY Id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", "1:30 2:30 3:25 4:25 5:5")] + [InlineData("COUNT(DISTINCT T)", "", "1:5 2:5 3:5 4:5 5:5")] + public void Distinct_counts_each_value_of_the_frame_once(string function, string over, string expected) => + Assert.Equal(expected, ById(function, over)); + + [Theory] + [InlineData("COUNT(DISTINCT V)")] + [InlineData("SUM(DISTINCT V)")] + [InlineData("AVG(DISTINCT V)")] + [InlineData("STDEV(DISTINCT M)")] + public void A_distinct_aggregate_over_the_whole_input_is_the_grouped_one(string aggregate) + { + var (groupedTypes, grouped) = Query($"SELECT {aggregate} AS r FROM W"); + var (windowedTypes, windowed) = Query($"SELECT {aggregate} OVER () AS r FROM W"); + + Assert.Equal(groupedTypes[0], windowedTypes[0]); + Assert.All(windowed, row => Assert.Equal(grouped[0][0], row[0])); + } + + [Theory] + [InlineData("FIRST_VALUE(DISTINCT V) OVER (ORDER BY Id)")] + [InlineData("LAG(DISTINCT V) OVER (ORDER BY Id)")] + [InlineData("NTILE(DISTINCT 2) OVER (ORDER BY Id)")] + public void Only_an_aggregate_takes_distinct(string expression) => + Assert.Throws(() => Query($"SELECT {expression} AS r FROM W")); + + [Fact] + public void Only_count_takes_a_star() => + Assert.Throws(() => Query("SELECT SUM(*) OVER () AS r FROM W")); +} diff --git a/test/LibRed.Engine.Tests/WindowFrameTests.cs b/test/LibRed.Engine.Tests/WindowFrameTests.cs new file mode 100644 index 000000000..c798b13b9 --- /dev/null +++ b/test/LibRed.Engine.Tests/WindowFrameTests.cs @@ -0,0 +1,124 @@ +using System.Globalization; +using LibRed.Sql.Parsing; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// Explicit window frames — ROWS, RANGE and GROUPS, with BETWEEN bounds and EXCLUDE +/// — as the SQL standard defines them. Access has no window functions; this is a LibRed extension. +/// +public class WindowFrameTests(WindowFrameTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Partition 'a' in V order is Ids 5 (Null), 1, 2 and 3 (tied on 2), 4; D has gaps of 1, 2 and 4 days. K has + // columns named with the frame clause's words, which are not reserved. + private static readonly string[] Setup = + [ + "CREATE TABLE F (Id LONG, G TEXT(10), V LONG, D DATETIME)", + "INSERT INTO F (Id, G, V, D) VALUES (1, 'a', 1, #2020-01-01#)", + "INSERT INTO F (Id, G, V, D) VALUES (2, 'a', 2, #2020-01-02#)", + "INSERT INTO F (Id, G, V, D) VALUES (3, 'a', 2, #2020-01-04#)", + "INSERT INTO F (Id, G, V, D) VALUES (4, 'a', 5, #2020-01-08#)", + "INSERT INTO F (Id, G, V, D) VALUES (5, 'a', NULL, NULL)", + "INSERT INTO F (Id, G, V, D) VALUES (6, 'b', 10, #2020-02-01#)", + "INSERT INTO F (Id, G, V, D) VALUES (7, 'b', 20, #2020-02-15#)", + "CREATE TABLE K (Range LONG, Current LONG, Groups LONG, Ties LONG)", + "INSERT INTO K (Range, Current, Groups, Ties) VALUES (1, 10, 100, 1000)", + "INSERT INTO K (Range, Current, Groups, Ties) VALUES (2, 20, 200, 2000)", + ]; + + public sealed class Database() : SharedDatabase("window-frame-", Setup); + + private string ById(string expression) => string.Join(" ", + database.Query($"SELECT Id, {expression} AS r FROM F ORDER BY Id", CultureInfo.InvariantCulture).Rows + .Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", "1:3 2:5 3:9 4:7 5:15 6:30 7:30")] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS UNBOUNDED PRECEDING)", "1:1 2:3 3:5 4:10 5:10 6:20 7:40")] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)", "1:40 2:39 3:37 4:35 5:30 6:30 7:20")] + [InlineData("COUNT(*) OVER (ORDER BY Id ROWS BETWEEN 3 PRECEDING AND 2 PRECEDING)", "1:0 2:0 3:1 4:2 5:2 6:2 7:2")] + [InlineData("MIN(V) OVER (ORDER BY Id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", "1:1 2:1 3:2 4:2 5:5 6:10 7:10")] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS BETWEEN 5 FOLLOWING AND 9 FOLLOWING)", "1:30 2:20 3: 4: 5: 6: 7:")] + [InlineData("COUNT(*) OVER (ORDER BY Id ROWS BETWEEN Id - 1 PRECEDING AND CURRENT ROW)", "1:1 2:2 3:3 4:4 5:5 6:6 7:7")] + public void Rows_frames_count_rows(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("SUM(V) OVER (PARTITION BY G ORDER BY V RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)", "1:5 2:5 3:5 4:5 5: 6:10 7:20")] + [InlineData("COUNT(*) OVER (PARTITION BY G ORDER BY V RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)", "1:1 2:2 3:2 4:4 5:1 6:0 7:1")] + [InlineData("SUM(V) OVER (PARTITION BY G ORDER BY V DESC RANGE BETWEEN CURRENT ROW AND 3 FOLLOWING)", "1:1 2:5 3:5 4:9 5: 6:10 7:20")] + [InlineData("SUM(V) OVER (ORDER BY V RANGE BETWEEN 0.5 PRECEDING AND 0.5 FOLLOWING)", "1:1 2:4 3:4 4:5 5: 6:10 7:20")] + [InlineData("COUNT(*) OVER (PARTITION BY G ORDER BY D RANGE 2 PRECEDING)", "1:1 2:2 3:2 4:1 5:1 6:1 7:1")] + [InlineData("SUM(V) OVER (PARTITION BY G ORDER BY V RANGE CURRENT ROW)", "1:1 2:4 3:4 4:5 5: 6:10 7:20")] + public void Range_frames_measure_the_order_by_key(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("SUM(V) OVER (PARTITION BY G ORDER BY V GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)", "1:1 2:5 3:5 4:9 5: 6:10 7:30")] + [InlineData("COUNT(*) OVER (PARTITION BY G ORDER BY V GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)", "1:3 2:1 3:1 4:0 5:4 6:1 7:0")] + public void Groups_frames_count_peer_groups(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + private const string Whole = "PARTITION BY G ORDER BY V ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"; + + [Theory] + [InlineData("SUM(V) OVER (" + Whole + ")", "1:10 2:10 3:10 4:10 5:10 6:30 7:30")] + [InlineData("SUM(V) OVER (" + Whole + " EXCLUDE NO OTHERS)", "1:10 2:10 3:10 4:10 5:10 6:30 7:30")] + [InlineData("SUM(V) OVER (" + Whole + " EXCLUDE CURRENT ROW)", "1:9 2:8 3:8 4:5 5:10 6:20 7:10")] + [InlineData("SUM(V) OVER (" + Whole + " EXCLUDE GROUP)", "1:9 2:6 3:6 4:5 5:10 6:20 7:10")] + [InlineData("SUM(V) OVER (" + Whole + " EXCLUDE TIES)", "1:10 2:8 3:8 4:10 5:10 6:30 7:30")] + [InlineData("NTH_VALUE(Id, 2) OVER (" + Whole + " EXCLUDE CURRENT ROW)", "1:2 2:1 3:1 4:1 5:2 6: 7:")] + public void Exclusion_leaves_rows_around_the_current_one_out(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("LAST_VALUE(Id) OVER (" + Whole + ")", "1:4 2:4 3:4 4:4 5:4 6:7 7:7")] + [InlineData("FIRST_VALUE(Id) OVER (ORDER BY Id ROWS BETWEEN 2 FOLLOWING AND 3 FOLLOWING)", "1:3 2:4 3:5 4:6 5:7 6: 7:")] + [InlineData("FIRST_VALUE(Id) OVER (PARTITION BY G ORDER BY V ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING EXCLUDE GROUP)", + "1:2 2:4 3:4 4: 5:1 6:7 7:")] + [InlineData("LAST(V) OVER (ORDER BY Id ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)", "1: 2:1 3:2 4:2 5:5 6: 7:10")] + public void Value_functions_read_the_frame(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Fact] + public void The_frame_words_still_name_columns() + { + var (_, rows) = database.Query( + "SELECT Range, SUM(Current) OVER (ORDER BY Range ROWS UNBOUNDED PRECEDING) AS Following, Groups, Ties FROM K ORDER BY Range", + CultureInfo.InvariantCulture); + Assert.Equal("1 10 100 1000|2 30 200 2000", string.Join("|", + rows.Select(row => string.Join(" ", row.Select(v => Convert.ToString(v, CultureInfo.InvariantCulture)))))); + } + + [Theory] + [InlineData("ROWS BETWEEN CURRENT ROW AND 1 PRECEDING")] + [InlineData("ROWS BETWEEN 1 FOLLOWING AND CURRENT ROW")] + [InlineData("ROWS 1 FOLLOWING")] + [InlineData("ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING")] + [InlineData("ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING")] + public void A_frame_that_starts_after_it_ends_is_a_syntax_error(string frame) => + Assert.Throws(() => ById($"SUM(V) OVER (ORDER BY Id {frame})")); + + [Theory] + [InlineData("ROW_NUMBER() OVER (ORDER BY Id ROWS UNBOUNDED PRECEDING)")] + [InlineData("LAG(V) OVER (ORDER BY Id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)")] + [InlineData("SUM(V) OVER (GROUPS UNBOUNDED PRECEDING)")] + [InlineData("SUM(V) OVER (ORDER BY G, V RANGE 1 PRECEDING)")] + [InlineData("SUM(V) OVER (RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)")] + public void A_frame_the_window_cannot_take_is_refused(string expression) => + Assert.Throws(() => ById(expression)); + + [Fact] + public void A_range_offset_over_text_is_a_type_mismatch() => + Assert.Throws(() => ById("SUM(V) OVER (ORDER BY G RANGE 1 PRECEDING)")); + + [Theory] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS -1 PRECEDING)")] + [InlineData("SUM(V) OVER (ORDER BY Id ROWS NULL PRECEDING)")] + [InlineData("SUM(V) OVER (ORDER BY Id GROUPS BETWEEN CURRENT ROW AND -2 FOLLOWING)")] + [InlineData("SUM(V) OVER (ORDER BY V RANGE -0.5 PRECEDING)")] + public void A_null_or_negative_offset_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => ById(expression)); +} diff --git a/test/LibRed.Engine.Tests/WindowFunctionTests.cs b/test/LibRed.Engine.Tests/WindowFunctionTests.cs index b8f2bdd6f..aed6ec7ee 100644 --- a/test/LibRed.Engine.Tests/WindowFunctionTests.cs +++ b/test/LibRed.Engine.Tests/WindowFunctionTests.cs @@ -133,17 +133,19 @@ public void Ranking_functions_share_a_rank_between_peers(string function, string public void An_unknown_window_function_is_reported_by_name() { var ex = Assert.ThrowsAny(() => Seeded().ExecuteQuery( - "SELECT NTILE(4) OVER (ORDER BY `Id`) AS `r` FROM `W`")); + "SELECT RATIO_TO_REPORT(`Id`) OVER (ORDER BY `Id`) AS `r` FROM `W`")); - Assert.Contains("NTILE", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("RATIO_TO_REPORT", ex.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public void A_window_over_a_grouped_query_is_refused() - // Not supported rather than silently wrong: AggregateNode owns the projection and collapses rows, so - // the window would have to be computed above it. Nothing EF Core emits needs this. - => Assert.ThrowsAny(() => Seeded().ExecuteQuery( - "SELECT COUNT(*) AS `c`, ROW_NUMBER() OVER (ORDER BY `G`) AS `r` FROM `W` GROUP BY `G`")); + public void A_window_over_a_grouped_query_numbers_the_groups() + // The groups are the window's rows: NULL (2 rows), 'a' (2) and 'b' (3), numbered in G order — Nulls first. + => Assert.Equal( + "2:1 2:2 3:3", + string.Join(" ", Seeded().ExecuteQuery( + "SELECT COUNT(*) AS `c`, ROW_NUMBER() OVER (ORDER BY `G`) AS `r` FROM `W` GROUP BY `G`") + .Rows.Select(r => $"{r[0]}:{r[1]}"))); [Fact] public void Index_selection_still_reaches_below_the_window() diff --git a/test/LibRed.Engine.Tests/WindowNullTreatmentTests.cs b/test/LibRed.Engine.Tests/WindowNullTreatmentTests.cs new file mode 100644 index 000000000..48cc4cb34 --- /dev/null +++ b/test/LibRed.Engine.Tests/WindowNullTreatmentTests.cs @@ -0,0 +1,95 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// RESPECT NULLS / IGNORE NULLS on LAG, LEAD, FIRST_VALUE, LAST_VALUE and +/// NTH_VALUE, and NTH_VALUE … FROM FIRST / FROM LAST, as the SQL standard defines them. Access +/// has no window functions; this is a LibRed extension. +/// +public class WindowNullTreatmentTests(WindowNullTreatmentTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // V by Id: 10, Null, Null, 40, Null | Null, 70 — partition 'a' is Ids 1-5, 'b' Ids 6-7. The table named Last and + // its columns carry the new clauses' words, which are not reserved. + private static readonly string[] Setup = + [ + "CREATE TABLE N (Id LONG, G TEXT(10), V LONG)", + "INSERT INTO N (Id, G, V) VALUES (1, 'a', 10)", + "INSERT INTO N (Id, G, V) VALUES (2, 'a', NULL)", + "INSERT INTO N (Id, G, V) VALUES (3, 'a', NULL)", + "INSERT INTO N (Id, G, V) VALUES (4, 'a', 40)", + "INSERT INTO N (Id, G, V) VALUES (5, 'a', NULL)", + "INSERT INTO N (Id, G, V) VALUES (6, 'b', NULL)", + "INSERT INTO N (Id, G, V) VALUES (7, 'b', 70)", + "CREATE TABLE Last (Respect LONG, Nulls LONG, Within LONG)", + "INSERT INTO Last (Respect, Nulls, Within) VALUES (1, 2, 3)", + "INSERT INTO Last (Respect, Nulls, Within) VALUES (4, 5, 6)", + ]; + + public sealed class Database() : SharedDatabase("window-nulls-", Setup); + + private string ById(string expression) => string.Join(" ", + database.Query($"SELECT Id, {expression} AS r FROM N ORDER BY Id", CultureInfo.InvariantCulture).Rows + .Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("LAG(V) OVER (ORDER BY Id)", "1: 2:10 3: 4: 5:40 6: 7:")] + [InlineData("LAG(V) RESPECT NULLS OVER (ORDER BY Id)", "1: 2:10 3: 4: 5:40 6: 7:")] + [InlineData("LAG(V) IGNORE NULLS OVER (ORDER BY Id)", "1: 2:10 3:10 4:10 5:40 6:40 7:40")] + [InlineData("LEAD(V) IGNORE NULLS OVER (ORDER BY Id)", "1:40 2:40 3:40 4:70 5:70 6:70 7:")] + [InlineData("LAG(V, 2, -1) IGNORE NULLS OVER (ORDER BY Id)", "1:-1 2:-1 3:-1 4:-1 5:10 6:10 7:10")] + [InlineData("LAG(V, 0) IGNORE NULLS OVER (ORDER BY Id)", "1:10 2: 3: 4:40 5: 6: 7:70")] + [InlineData("LAG(V) IGNORE NULLS OVER (PARTITION BY G ORDER BY Id)", "1: 2:10 3:10 4:10 5:40 6: 7:")] + public void Lag_and_lead_can_skip_rows_whose_value_is_null(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("FIRST_VALUE(V) OVER (PARTITION BY G ORDER BY Id)", "1:10 2:10 3:10 4:10 5:10 6: 7:")] + [InlineData("FIRST_VALUE(V) IGNORE NULLS OVER (PARTITION BY G ORDER BY Id)", "1:10 2:10 3:10 4:10 5:10 6: 7:70")] + [InlineData("LAST_VALUE(V) IGNORE NULLS OVER (ORDER BY Id ROWS UNBOUNDED PRECEDING)", "1:10 2:10 3:10 4:40 5:40 6:40 7:70")] + [InlineData("FIRST_VALUE(V) IGNORE NULLS OVER (ORDER BY Id ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)", + "1:10 2:40 3:40 4:40 5:70 6:70 7:70")] + [InlineData("NTH_VALUE(V, 2) IGNORE NULLS OVER (ORDER BY Id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + "1:40 2:40 3:40 4:40 5:40 6:40 7:40")] + public void Frame_values_can_skip_rows_whose_value_is_null(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("NTH_VALUE(Id, 2) FROM LAST OVER (ORDER BY Id)", "1: 2:1 3:2 4:3 5:4 6:5 7:6")] + [InlineData("NTH_VALUE(Id, 1) FROM FIRST OVER (ORDER BY Id)", "1:1 2:1 3:1 4:1 5:1 6:1 7:1")] + [InlineData("NTH_VALUE(Id, 2) FROM LAST OVER (ORDER BY Id ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + "1:6 2:6 3:6 4:6 5:6 6:6 7:6")] + [InlineData("NTH_VALUE(V, 1) FROM LAST IGNORE NULLS OVER (ORDER BY Id)", "1:10 2:10 3:10 4:40 5:40 6:40 7:70")] + [InlineData("NTH_VALUE(V, 2) FROM LAST IGNORE NULLS OVER (ORDER BY Id)", "1: 2: 3: 4:10 5:10 6:10 7:40")] + public void Nth_value_counts_from_either_end_of_the_frame(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + private const string Whole = "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"; + + // Ordered by G, Ids 1-5 and 6-7 are peers; EXCLUDE TIES keeps the row itself but drops the rest of its group. + [Theory] + [InlineData("FIRST_VALUE(V) IGNORE NULLS OVER (ORDER BY Id " + Whole + " EXCLUDE CURRENT ROW)", "1:40 2:10 3:10 4:10 5:10 6:10 7:10")] + [InlineData("LAST_VALUE(V) IGNORE NULLS OVER (ORDER BY Id " + Whole + " EXCLUDE CURRENT ROW)", "1:70 2:70 3:70 4:70 5:70 6:70 7:40")] + [InlineData("FIRST_VALUE(V) IGNORE NULLS OVER (ORDER BY G " + Whole + " EXCLUDE TIES)", "1:10 2:70 3:70 4:40 5:70 6:10 7:10")] + [InlineData("LAST_VALUE(V) IGNORE NULLS OVER (ORDER BY G " + Whole + " EXCLUDE TIES)", "1:70 2:70 3:70 4:70 5:70 6:40 7:70")] + public void Skipping_nulls_works_across_an_exclusion(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("SUM(V) IGNORE NULLS OVER ()")] + [InlineData("ROW_NUMBER() RESPECT NULLS OVER (ORDER BY Id)")] + [InlineData("FIRST_VALUE(V) FROM LAST OVER ()")] + [InlineData("LAG(V) FROM FIRST OVER (ORDER BY Id)")] + public void Only_the_functions_the_standard_names_take_the_clauses(string expression) => + Assert.Throws(() => ById(expression)); + + [Fact] + public void The_clauses_words_still_name_tables_and_columns() + { + Assert.Equal(4, database.Scalar("SELECT MAX(Respect) FROM Last", CultureInfo.InvariantCulture)); + var (_, rows) = database.Query("SELECT Respect, Nulls, Within FROM Last ORDER BY Respect", CultureInfo.InvariantCulture); + Assert.Equal([1, 2, 3], rows[0]); + } +} diff --git a/test/LibRed.Engine.Tests/WindowRankAndOffsetTests.cs b/test/LibRed.Engine.Tests/WindowRankAndOffsetTests.cs new file mode 100644 index 000000000..10c7c22a1 --- /dev/null +++ b/test/LibRed.Engine.Tests/WindowRankAndOffsetTests.cs @@ -0,0 +1,128 @@ +using System.Globalization; +using Xunit; + +namespace LibRed.Engine.Tests; + +/// +/// NTILE, PERCENT_RANK, CUME_DIST, LAG, LEAD, FIRST_VALUE, LAST_VALUE +/// and NTH_VALUE, as the SQL standard defines them, and Access's First and Last over a window. +/// Access has no window functions; this is a LibRed extension. +/// +public class WindowRankAndOffsetTests(WindowRankAndOffsetTests.Database database) + : TempDatabaseTest, IClassFixture +{ + // Partition 'a' has a tie on V (Ids 2 and 3); partition 'b' has a Null V. + private static readonly string[] Setup = + [ + "CREATE TABLE W (Id LONG, G TEXT(10), V LONG, M CURRENCY)", + "INSERT INTO W (Id, G, V, M) VALUES (1, 'a', 10, 1.5)", + "INSERT INTO W (Id, G, V, M) VALUES (2, 'a', 20, 2.25)", + "INSERT INTO W (Id, G, V, M) VALUES (3, 'a', 20, 3)", + "INSERT INTO W (Id, G, V, M) VALUES (4, 'a', 40, 4)", + "INSERT INTO W (Id, G, V, M) VALUES (5, 'b', 5, 5)", + "INSERT INTO W (Id, G, V, M) VALUES (6, 'b', NULL, 6)", + ]; + + public sealed class Database() : SharedDatabase("window-rank-offset-", Setup); + + private (IReadOnlyList Types, List Rows) Query(string expression) => + database.Query($"SELECT Id, {expression} AS r FROM W ORDER BY Id", CultureInfo.InvariantCulture); + + private string ById(string expression) => string.Join(" ", + Query(expression).Rows.Select(row => $"{row[0]}:{Convert.ToString(row[1], CultureInfo.InvariantCulture)}")); + + [Theory] + [InlineData("NTILE(4) OVER (ORDER BY Id)", "1:1 2:1 3:2 4:2 5:3 6:4")] + [InlineData("NTILE(2) OVER (PARTITION BY G ORDER BY Id)", "1:1 2:1 3:2 4:2 5:1 6:2")] + [InlineData("NTILE(10) OVER (ORDER BY Id)", "1:1 2:2 3:3 4:4 5:5 6:6")] + [InlineData("NTILE(2.5) OVER (ORDER BY Id)", "1:1 2:1 3:1 4:2 5:2 6:2")] + [InlineData("NTILE(4) OVER (ORDER BY Id DESC)", "1:4 2:3 3:2 4:2 5:1 6:1")] + [InlineData("NTILE(NULL) OVER (ORDER BY Id)", "1: 2: 3: 4: 5: 6:")] + public void Ntile_cuts_the_partition_into_buckets(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("PERCENT_RANK() OVER (PARTITION BY G ORDER BY V)", "1:0 2:0.3333333333333333 3:0.3333333333333333 4:1 5:1 6:0")] + [InlineData("PERCENT_RANK() OVER (PARTITION BY G)", "1:0 2:0 3:0 4:0 5:0 6:0")] + [InlineData("CUME_DIST() OVER (PARTITION BY G ORDER BY V)", "1:0.25 2:0.75 3:0.75 4:1 5:1 6:0.5")] + [InlineData("CUME_DIST() OVER ()", "1:1 2:1 3:1 4:1 5:1 6:1")] + public void Distribution_functions_count_peers_together(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Fact] + public void A_partition_of_one_has_a_percent_rank_of_zero() => + Assert.Equal("1:0 2:0 3:0 4:0 5:0 6:0", ById("PERCENT_RANK() OVER (PARTITION BY Id)")); + + [Theory] + [InlineData("LAG(V) OVER (PARTITION BY G ORDER BY Id)", "1: 2:10 3:20 4:20 5: 6:5")] + [InlineData("LEAD(V) OVER (PARTITION BY G ORDER BY Id)", "1:20 2:20 3:40 4: 5: 6:")] + [InlineData("LAG(V, 2, 0) OVER (ORDER BY Id)", "1:0 2:0 3:10 4:20 5:20 6:40")] + [InlineData("LEAD(V, 0) OVER (ORDER BY Id)", "1:10 2:20 3:20 4:40 5:5 6:")] + [InlineData("LEAD(V, 1, -1) OVER (ORDER BY V)", "1:20 2:20 3:40 4:-1 5:10 6:5")] + [InlineData("LAG(V, Id - 1) OVER (ORDER BY Id)", "1:10 2:10 3:10 4:10 5:10 6:10")] + [InlineData("LAG(V, NULL) OVER (ORDER BY Id)", "1: 2: 3: 4: 5: 6:")] + [InlineData("LAG(V, 1, Id * 100) OVER (PARTITION BY G ORDER BY Id)", "1:100 2:10 3:20 4:20 5:500 6:5")] + public void Lag_and_lead_read_another_row_of_the_partition(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("FIRST_VALUE(Id) OVER (PARTITION BY G ORDER BY V)", "1:1 2:1 3:1 4:1 5:6 6:6")] + [InlineData("FIRST_VALUE(V) OVER (PARTITION BY G ORDER BY V)", "1:10 2:10 3:10 4:10 5: 6:")] + [InlineData("LAST_VALUE(Id) OVER (PARTITION BY G ORDER BY V)", "1:1 2:3 3:3 4:4 5:5 6:6")] + [InlineData("LAST_VALUE(Id) OVER (PARTITION BY G)", "1:4 2:4 3:4 4:4 5:6 6:6")] + [InlineData("NTH_VALUE(Id, 2) OVER (PARTITION BY G ORDER BY V)", "1: 2:2 3:2 4:2 5:5 6:")] + [InlineData("NTH_VALUE(Id, 3) OVER (ORDER BY Id)", "1: 2: 3:3 4:3 5:3 6:3")] + [InlineData("NTH_VALUE(Id, 5) OVER (PARTITION BY G)", "1: 2: 3: 4: 5: 6:")] + [InlineData("NTH_VALUE(Id, NULL) OVER (ORDER BY Id)", "1: 2: 3: 4: 5: 6:")] + [InlineData("FIRST(V) OVER (PARTITION BY G ORDER BY Id DESC)", "1:40 2:40 3:40 4:40 5: 6:")] + [InlineData("LAST(M) OVER (ORDER BY Id)", "1:1.5 2:2.25 3:3 4:4 5:5 6:6")] + public void Value_functions_read_a_row_of_the_frame(string expression, string expected) => + Assert.Equal(expected, ById(expression)); + + [Theory] + [InlineData("FIRST_VALUE(M) OVER (ORDER BY Id)", typeof(decimal))] + [InlineData("LAST_VALUE(G) OVER (ORDER BY Id)", typeof(string))] + [InlineData("NTH_VALUE(V, 2) OVER (ORDER BY Id)", typeof(int))] + [InlineData("NTILE(4) OVER (ORDER BY Id)", typeof(int))] + [InlineData("PERCENT_RANK() OVER (ORDER BY Id)", typeof(double))] + [InlineData("CUME_DIST() OVER (ORDER BY Id)", typeof(double))] + [InlineData("LAG(V) OVER (ORDER BY Id)", typeof(int))] + [InlineData("LEAD(M) OVER (ORDER BY Id)", typeof(decimal))] + [InlineData("LAG(V, 1, 0.5) OVER (ORDER BY Id)", typeof(decimal))] // a written decimal is a Decimal to the ladder + [InlineData("LAG(V, 1, CDBL(0.5)) OVER (ORDER BY Id)", typeof(double))] + [InlineData("LAG(M, 1, 0) OVER (ORDER BY Id)", typeof(decimal))] + [InlineData("LAG(V, 1, NULL) OVER (ORDER BY Id)", typeof(int))] + public void Every_value_has_the_declared_type(string expression, Type expected) + { + var (types, rows) = Query(expression); + Assert.Equal(expected, types[1]); + Assert.All(rows, row => Assert.True(row[1] is null || row[1]!.GetType() == expected)); + } + + [Fact] + public void A_default_of_another_kind_leaves_the_column_untyped() + { + var (types, rows) = Query("LAG(V, 1, 'none') OVER (ORDER BY Id)"); + Assert.Equal(typeof(object), types[1]); + Assert.Equal("none", rows[0][1]); + Assert.Equal(10, rows[1][1]); + } + + [Theory] + [InlineData("NTILE(0) OVER (ORDER BY Id)")] + [InlineData("NTILE(-1) OVER (ORDER BY Id)")] + [InlineData("LAG(V, -1) OVER (ORDER BY Id)")] + [InlineData("LEAD(V, -2, 0) OVER (ORDER BY Id)")] + [InlineData("NTH_VALUE(V, 0) OVER (ORDER BY Id)")] + public void A_count_or_offset_out_of_range_is_an_invalid_procedure_call(string expression) => + Assert.Throws(() => Query(expression)); + + [Theory] + [InlineData("NTILE() OVER (ORDER BY Id)")] + [InlineData("LAG() OVER (ORDER BY Id)")] + [InlineData("LAG(V, 1, 0, 0) OVER (ORDER BY Id)")] + [InlineData("PERCENT_RANK(V) OVER (ORDER BY Id)")] + [InlineData("NTH_VALUE(V) OVER (ORDER BY Id)")] + public void The_wrong_number_of_arguments_is_refused(string expression) => + Assert.Throws(() => Query(expression)); +} diff --git a/test/LibRed.Shared/AceTestDatabase.cs b/test/LibRed.Shared/AceTestDatabase.cs index 28fe122d4..acc316d4a 100644 --- a/test/LibRed.Shared/AceTestDatabase.cs +++ b/test/LibRed.Shared/AceTestDatabase.cs @@ -16,6 +16,9 @@ public static OleDbConnection Open(string path, string? password = null, int att ArgumentException.ThrowIfNullOrEmpty(path); if (attempts < 1) throw new ArgumentOutOfRangeException(nameof(attempts)); + // A test that drove DAO first may have left its temporaries to the finalizer; see ReleaseAbandonedComObjects. + ReleaseAbandonedComObjects(); + Exception? last = null; for (int attempt = 0; attempt < attempts; attempt++) { @@ -55,6 +58,47 @@ public static OleDbConnection Open(string path, string? password = null, int att throw new InvalidOperationException("No Microsoft ACE OLE DB provider could open the test database.", last); } + /// + /// Releases every COM object — DAO's or the ACE OLE DB provider's — that code so far has abandoned to the + /// finalizer, and returns only once that is done. + /// + /// + /// ACE faults when two threads are inside it at once (see AceCollection), and serialising the tests + /// does not stop that on its own. DAO is apartment-threaded, so an engine created from a test's thread lives + /// on a COM-created thread of its own, and the probes release none of what they create. Each Workspace, + /// Database, TableDef and Field is torn down when the finalizer gets to it — inside ACE, on that COM thread, + /// at whatever moment a GC happens to run, which is usually in the middle of a later test that is itself + /// inside ACE. An OLE DB object left undisposed does the same the other way round, from the finalizer thread + /// into a later DAO call. + /// What that looks like: RPC_E_SERVERFAULT out of a DAO call, then the next ACE test hanging + /// until the blame collector kills the host — in a different test from run to run, because it depends on + /// when the GC runs, and seen on CI's runners only. Doing the teardown here, at points where no test is + /// inside ACE, is what takes it off that timing. + /// + public static void ReleaseAbandonedComObjects() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + // Finalizing an RCW can free what kept another one alive; the second pass collects those. + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + /// A DAO DBEngine, from the newest version registered in this bitness; null when there is none. + /// Releases whatever the test has abandoned first (see ). + public static object? CreateDaoEngine() + { + ReleaseAbandonedComObjects(); + foreach (int version in new[] { 170, 160, 150, 140, 130, 120 }) + { + Type? type = Type.GetTypeFromProgID($"DAO.DBEngine.{version}"); + if (type is null) continue; + try { return Activator.CreateInstance(type); } + catch (Exception) { /* registered but not instantiable in this bitness */ } + } + return null; + } + private static readonly Dictionary ColumnTypeSupport = []; ///