diff --git a/CHANGELOG.md b/CHANGELOG.md index 7842acc..b44926d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,14 @@ All notable changes to this project will be documented in this file. It uses the ## [v0.1.1] — Unreleased +### ⚡ Improvements + +* Support mapping `BFloat16` to Postgres `real`, and `Interval` types to + Postgres `interval` ([#76]). + ### 🐞 Bug Fixes +* `DateTime64` or `Time64` now defaults to millisecond precision ([#76]). * Added the `date_time_output_format='iso'` setting to each chDB query to always format `timestamp` and `timestamptz` values in plain text formats with the ISO-8601 format in UTC, converting `timestamp` values from the @@ -30,7 +36,9 @@ All notable changes to this project will be documented in this file. It uses the exported and imported values. [v0.1.1]: https://github.com/clickhouse/pg_chdb/compare/v0.1.0...v0.1.1 - [chdb_hook docs](./doc/chdb_hook.md) + [#76]: https://github.com/clickhouse/pg_chdb/issues/76 + [chdb_hook docs]: ./doc/chdb_hook.md + [structure]: ./doc/chdb_hook.md#structure "chdb_hook Docs: structure" ## [v0.1.0] — 2026-08-25 diff --git a/doc/chdb_hook.md b/doc/chdb_hook.md index b142eb3..678a0a4 100644 --- a/doc/chdb_hook.md +++ b/doc/chdb_hook.md @@ -454,7 +454,7 @@ those you need. | cidr | String | | | macaddr | String | | | macaddr8 | String | | -| interval | String | | +| interval | String | Override with an `Interval` unit such as `IntervalDay`. | | tsvector | String | | | tsquery | String | | | jsonpath | String | | @@ -537,53 +537,65 @@ chdb_hook maps the ClickHouse types reported by [`DESCRIBE`] to these Postgres types: -| chDB | Postgres | Notes | -|-------------------|-----------------------------|----------------------------------| -| Array(T) | T[] | One PG array type per depth | -| Bool | boolean | | -| Date | date | | -| Date32 | date | | -| DateTime | timestamp with time zone | | -| DateTime64(P) | timestamp(P) with time zone | P over 6 caps at 6 | -| Decimal(P,S) | numeric(P,S) | | -| Decimal32(S) | numeric(9,S) | | -| Decimal64(S) | numeric(18,S) | | -| Decimal128(S) | numeric(38,S) | | -| Decimal256(S) | numeric(76,S) | | -| Enum8 | text | | -| Enum16 | text | | -| FixedString(N) | text | N counts CH bytes, PG characters | -| Float32 | real | | -| Float64 | double precision | | -| IPv4 | inet | | -| IPv6 | inet | | -| Int8 | smallint | | -| Int16 | smallint | | -| Int32 | integer | | -| Int64 | bigint | | -| JSON | jsonb | Also reads into json | -| LineString | path | | -| LowCardinality(T) | T | | -| Map(K,V) | text[][] | One row of text items per pair | -| MultiLineString | path[] | | -| MultiPolygon | polygon[][] | | -| Nullable(T) | T | Sets nullable on the column | -| Point | point | | -| Polygon | polygon[] | | -| Ring | polygon | | -| String | text | Also reads into bytea | -| Time | time without time zone | | -| Time64(P) | time(P) without time zone | P over 6 caps at 6 | -| Tuple(...) | text[] | Fields become text items | -| UInt8 | smallint | | -| UInt16 | integer | | -| UInt32 | bigint | | -| UInt64 | bigint | Errors on values > BIGINT max | -| UUID | uuid | | +| chDB | Postgres | Notes | +|---------------------|-----------------------------|----------------------------------| +| Array(T) | T[] | One PG array type per depth | +| BFloat16 | real | Write drops low mantissa bits | +| Bool | boolean | | +| Date | date | | +| Date32 | date | | +| DateTime | timestamp with time zone | | +| DateTime64(P) | timestamp(P) with time zone | P over 6 caps at 6 | +| Decimal(P,S) | numeric(P,S) | | +| Decimal32(S) | numeric(9,S) | | +| Decimal64(S) | numeric(18,S) | | +| Decimal128(S) | numeric(38,S) | | +| Decimal256(S) | numeric(76,S) | | +| Enum8 | text | | +| Enum16 | text | | +| FixedString(N) | text | N counts CH bytes, PG characters | +| Float32 | real | | +| Float64 | double precision | | +| IPv4 | inet | | +| IPv6 | inet | | +| Int8 | smallint | | +| Int16 | smallint | | +| Int32 | integer | | +| Int64 | bigint | | +| IntervalDay | interval | | +| IntervalHour | interval | | +| IntervalMicrosecond | interval | | +| IntervalMillisecond | interval | | +| IntervalMinute | interval | | +| IntervalMonth | interval | | +| IntervalNanosecond | interval | Truncates to microsecond | +| IntervalQuarter | interval | | +| IntervalSecond | interval | | +| IntervalWeek | interval | | +| IntervalYear | interval | | +| JSON | jsonb | Also reads into json | +| LineString | path | | +| LowCardinality(T) | T | | +| Map(K,V) | text[][] | One row of text items per pair | +| MultiLineString | path[] | | +| MultiPolygon | polygon[][] | | +| Nullable(T) | T | Sets nullable on the column | +| Point | point | | +| Polygon | polygon[] | | +| Ring | polygon | | +| String | text | Also reads into bytea | +| Time | time without time zone | | +| Time64(P) | time(P) without time zone | P over 6 caps at 6 | +| Tuple(...) | text[] | Fields become text items | +| UInt8 | smallint | | +| UInt16 | integer | | +| UInt32 | bigint | | +| UInt64 | bigint | Errors on values > BIGINT max | +| UUID | uuid | | Every chDB type omitted from this table raises an error, among them `Nested`, -`Variant`, `Dynamic`, `Interval`, and the 128 and 256 bit integers. Use a +`Variant`, `Dynamic`, and the 128 and 256 bit integers. Use a [structure](#structure) that maps them to `String` to read them as text. Postgres holds a narrower range than chDB in a few of these types; thus copy diff --git a/test/expected/datetimes.out b/test/expected/datetimes.out index 3c52af9..ea44a6a 100644 --- a/test/expected/datetimes.out +++ b/test/expected/datetimes.out @@ -234,4 +234,64 @@ psql:test/utils/round-trip-formats.sql:421: STATEMENT: COPY "datetime_arrays" T f: ProtobufList t: MsgPack t: BSONEachRow +/****************************************************************************/ +CREATE TABLE intervals ( + y INTERVAL NOT NULL, + q INTERVAL NOT NULL, + mon INTERVAL NOT NULL, + w INTERVAL NOT NULL, + d INTERVAL NOT NULL, + h INTERVAL NOT NULL, + mi INTERVAL NOT NULL, + s INTERVAL NOT NULL, + ms INTERVAL NOT NULL, + us INTERVAL NOT NULL, + ns INTERVAL NOT NULL +); +INSERT INTO intervals +VALUES ('6 years', '9 months', '-5 months', '14 days', '-3 days', '4 hours', '-90 minutes', '2 sec', '0.25 sec', '-0.000001 sec', '1.000002 sec'); +\set interval_structure 'y IntervalYear, q IntervalQuarter, mon IntervalMonth, w IntervalWeek, d IntervalDay, h IntervalHour, mi IntervalMinute, s IntervalSecond, ms IntervalMillisecond, us IntervalMicrosecond, ns IntervalNanosecond' +CREATE TABLE intervals2 (LIKE intervals INCLUDING ALL); +COPY intervals TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure :'interval_structure'); +COPY intervals2 FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure :'interval_structure'); +SELECT count(*) AS intervals_mismatch + FROM (SELECT * FROM intervals EXCEPT ALL SELECT * FROM intervals2) x; + intervals_mismatch +-------------------- + 0 +(1 row) + +-- Months mixed with days, or a value which doesn't fit destination interval type. +CREATE TABLE misfits (iv INTERVAL NOT NULL); +INSERT INTO misfits VALUES ('1 mon 1 day'); +COPY misfits TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'iv IntervalMonth'); +ERROR: chdb: interval does not fit IntervalMonth +TRUNCATE misfits; +INSERT INTO misfits VALUES ('1.5 days'); +COPY misfits TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'iv IntervalDay'); +ERROR: chdb: interval does not fit IntervalDay +-- Reading same ticks truncates nanoseconds to microseconds in Postgres +CREATE TABLE ticks (n INT8 NOT NULL); +CREATE TABLE tocks (iv INTERVAL NOT NULL); +INSERT INTO ticks VALUES (1500), (-1500); +COPY ticks TO 'file:///tmp/datetimes.tmp' (format 'TabSeparated', structure 'n Int64'); +COPY tocks FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'n IntervalNanosecond'); +SELECT iv FROM tocks ORDER BY iv; + iv +--------------------- + @ 0.000001 secs ago + @ 0.000001 secs +(2 rows) + +TRUNCATE ticks; +INSERT INTO ticks VALUES (9223372036854775807); +COPY ticks TO 'file:///tmp/datetimes.tmp' (format 'TabSeparated', structure 'n Int64'); +COPY tocks FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'n IntervalYear'); +ERROR: chdb: IntervalYear value out of range \set ECHO errors diff --git a/test/expected/numbers.out b/test/expected/numbers.out index 0e536b9..f7bff41 100644 --- a/test/expected/numbers.out +++ b/test/expected/numbers.out @@ -145,4 +145,26 @@ f: Protobuf f: ProtobufList t: MsgPack t: BSONEachRow +/****************************************************************************/ +-- BFloat16 keeps the leading 16 bits of a float4, so a write drops the low +-- mantissa bits. +CREATE TABLE bfloats (f FLOAT4 NOT NULL); +INSERT INTO bfloats +VALUES (0), (1.5), (-2.25), (1.1), (3.4028235e38), ('infinity'), ('-infinity'), ('nan'); +CREATE TABLE bfloats2 (LIKE bfloats INCLUDING ALL); +COPY bfloats TO 'file:///tmp/numbers.tmp' (format 'TabSeparated', structure 'f BFloat16'); +COPY bfloats2 FROM 'file:///tmp/numbers.tmp' (format 'TabSeparated', structure 'f BFloat16'); +SELECT f FROM bfloats2 ORDER BY f; + f +--------------- + -Infinity + -2.25 + 0 + 1.09375 + 1.5 + 3.3895314e+38 + Infinity + NaN +(8 rows) + \set ECHO errors diff --git a/test/expected/timestamp.out b/test/expected/timestamp.out index 3fe1984..fbf3d11 100644 --- a/test/expected/timestamp.out +++ b/test/expected/timestamp.out @@ -146,4 +146,24 @@ SELECT pg_read_file(:'tsv_path'); (1 row) +/************************ Default scales ************************/ +-- A DateTime64 or Time64 named without a scale takes chDB's default of +-- milliseconds, truncating the microseconds Postgres holds. +CREATE TABLE default_scales ( + tstz TIMESTAMPTZ NOT NULL, + t TIME NOT NULL +); +INSERT INTO default_scales +VALUES ('2026-07-23 20:43:50.944042Z', '13:45:15.416013'); +COPY default_scales TO :'tsv_url' (structure 'tstz DateTime64, t Time64'); +TRUNCATE default_scales; +COPY default_scales FROM :'tsv_url' (structure 'tstz DateTime64, t Time64'); +SELECT tstz = '2026-07-23 20:43:50.944Z'::timestamptz AS tstz_ms + , t = '13:45:15.416'::time AS time_ms + FROM default_scales; + tstz_ms | time_ms +---------+--------- + t | t +(1 row) + \! rm -rf /tmp/chdb-timestamp.tmp 2> /dev/null || true diff --git a/test/sql/datetimes.sql b/test/sql/datetimes.sql index 340966b..7a582d0 100644 --- a/test/sql/datetimes.sql +++ b/test/sql/datetimes.sql @@ -66,5 +66,60 @@ CREATE TABLE datetime_arrays2 (LIKE datetime_arrays INCLUDING ALL); \set from_table datetime_arrays \set to_table datetime_arrays2 \i test/utils/round-trip-formats.sql + +/****************************************************************************/ +CREATE TABLE intervals ( + y INTERVAL NOT NULL, + q INTERVAL NOT NULL, + mon INTERVAL NOT NULL, + w INTERVAL NOT NULL, + d INTERVAL NOT NULL, + h INTERVAL NOT NULL, + mi INTERVAL NOT NULL, + s INTERVAL NOT NULL, + ms INTERVAL NOT NULL, + us INTERVAL NOT NULL, + ns INTERVAL NOT NULL +); + +INSERT INTO intervals +VALUES ('6 years', '9 months', '-5 months', '14 days', '-3 days', '4 hours', '-90 minutes', '2 sec', '0.25 sec', '-0.000001 sec', '1.000002 sec'); + +\set interval_structure 'y IntervalYear, q IntervalQuarter, mon IntervalMonth, w IntervalWeek, d IntervalDay, h IntervalHour, mi IntervalMinute, s IntervalSecond, ms IntervalMillisecond, us IntervalMicrosecond, ns IntervalNanosecond' + +CREATE TABLE intervals2 (LIKE intervals INCLUDING ALL); +COPY intervals TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure :'interval_structure'); +COPY intervals2 FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure :'interval_structure'); + +SELECT count(*) AS intervals_mismatch + FROM (SELECT * FROM intervals EXCEPT ALL SELECT * FROM intervals2) x; + +-- Months mixed with days, or a value which doesn't fit destination interval type. +CREATE TABLE misfits (iv INTERVAL NOT NULL); +INSERT INTO misfits VALUES ('1 mon 1 day'); +COPY misfits TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'iv IntervalMonth'); +TRUNCATE misfits; +INSERT INTO misfits VALUES ('1.5 days'); +COPY misfits TO 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'iv IntervalDay'); + +-- Reading same ticks truncates nanoseconds to microseconds in Postgres +CREATE TABLE ticks (n INT8 NOT NULL); +CREATE TABLE tocks (iv INTERVAL NOT NULL); +INSERT INTO ticks VALUES (1500), (-1500); +COPY ticks TO 'file:///tmp/datetimes.tmp' (format 'TabSeparated', structure 'n Int64'); +COPY tocks FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'n IntervalNanosecond'); +SELECT iv FROM tocks ORDER BY iv; + +TRUNCATE ticks; +INSERT INTO ticks VALUES (9223372036854775807); +COPY ticks TO 'file:///tmp/datetimes.tmp' (format 'TabSeparated', structure 'n Int64'); +COPY tocks FROM 'file:///tmp/datetimes.tmp' + (format 'TabSeparated', structure 'n IntervalYear'); + \set ECHO errors \! rm -rf /tmp/datetimes.tmp 2> /dev/null || true diff --git a/test/sql/numbers.sql b/test/sql/numbers.sql index 82e0753..69cdf70 100644 --- a/test/sql/numbers.sql +++ b/test/sql/numbers.sql @@ -75,5 +75,19 @@ CREATE TABLE number_arrays2 (LIKE number_arrays INCLUDING ALL); \set from_table number_arrays \set to_table number_arrays2 \i test/utils/round-trip-formats.sql +/****************************************************************************/ +-- BFloat16 keeps the leading 16 bits of a float4, so a write drops the low +-- mantissa bits. +CREATE TABLE bfloats (f FLOAT4 NOT NULL); + +INSERT INTO bfloats +VALUES (0), (1.5), (-2.25), (1.1), (3.4028235e38), ('infinity'), ('-infinity'), ('nan'); + +CREATE TABLE bfloats2 (LIKE bfloats INCLUDING ALL); +COPY bfloats TO 'file:///tmp/numbers.tmp' (format 'TabSeparated', structure 'f BFloat16'); +COPY bfloats2 FROM 'file:///tmp/numbers.tmp' (format 'TabSeparated', structure 'f BFloat16'); + +SELECT f FROM bfloats2 ORDER BY f; + \set ECHO errors \! rm -rf /tmp/numbers.tmp 2> /dev/null || true diff --git a/test/sql/timestamp.sql b/test/sql/timestamp.sql index a41fa84..423c599 100644 --- a/test/sql/timestamp.sql +++ b/test/sql/timestamp.sql @@ -76,5 +76,24 @@ SELECT pg_read_file(:'tsv_path'); COPY date_times TO :'tsv_url' (structure 'id Int8, ts DateTime64(3, ''Japan''), tstz DateTime64(3, ''America/New_York'')'); SELECT pg_read_file(:'tsv_path'); +/************************ Default scales ************************/ +-- A DateTime64 or Time64 named without a scale takes chDB's default of +-- milliseconds, truncating the microseconds Postgres holds. +CREATE TABLE default_scales ( + tstz TIMESTAMPTZ NOT NULL, + t TIME NOT NULL +); + +INSERT INTO default_scales +VALUES ('2026-07-23 20:43:50.944042Z', '13:45:15.416013'); + +COPY default_scales TO :'tsv_url' (structure 'tstz DateTime64, t Time64'); +TRUNCATE default_scales; +COPY default_scales FROM :'tsv_url' (structure 'tstz DateTime64, t Time64'); + +SELECT tstz = '2026-07-23 20:43:50.944Z'::timestamptz AS tstz_ms + , t = '13:45:15.416'::time AS time_ms + FROM default_scales; + \! rm -rf /tmp/chdb-timestamp.tmp 2> /dev/null || true diff --git a/vendor/pg-clickhouse-c b/vendor/pg-clickhouse-c index 609af0a..d2f127b 160000 --- a/vendor/pg-clickhouse-c +++ b/vendor/pg-clickhouse-c @@ -1 +1 @@ -Subproject commit 609af0a216707ca349edc2fd40ea13199661e4b0 +Subproject commit d2f127ba7e9e17dfa8785e40b34bb6be86b9df21