|
23 | 23 | import lombok.NoArgsConstructor; |
24 | 24 | import org.apache.commons.lang3.StringUtils; |
25 | 25 | import org.testng.Assert; |
| 26 | +import org.testng.SkipException; |
26 | 27 | import org.testng.annotations.AfterMethod; |
27 | 28 | import org.testng.annotations.BeforeMethod; |
28 | 29 | import org.testng.annotations.DataProvider; |
@@ -154,6 +155,183 @@ public static String tblCreateSQL(String table) { |
154 | 155 | } |
155 | 156 | } |
156 | 157 |
|
| 158 | + @Data |
| 159 | + @AllArgsConstructor |
| 160 | + @NoArgsConstructor |
| 161 | + public static class DTOForBFloat16Tests { |
| 162 | + private int rowId; |
| 163 | + private float bFloat16; |
| 164 | + private Float bFloat16Nullable; |
| 165 | + |
| 166 | + public static String tblCreateSQL(String table) { |
| 167 | + return tableDefinition(table, "rowId Int32", "bFloat16 BFloat16", |
| 168 | + "bFloat16Nullable Nullable(BFloat16)"); |
| 169 | + } |
| 170 | + } |
| 171 | + |
| 172 | + // BFloat16 was introduced in ClickHouse 24.11. |
| 173 | + private static final String BFLOAT16_UNSUPPORTED_VERSIONS = "(,24.10]"; |
| 174 | + |
| 175 | + @Test(groups = {"integration"}) |
| 176 | + public void testBFloat16() throws Exception { |
| 177 | + if (isVersionMatch(BFLOAT16_UNSUPPORTED_VERSIONS)) { |
| 178 | + throw new SkipException("BFloat16 requires ClickHouse 24.11+"); |
| 179 | + } |
| 180 | + |
| 181 | + // Exhaustively cover every one of the 2^16 BFloat16 bit patterns through a full client |
| 182 | + // write -> server -> client read round-trip (POJO path, incl. Nullable(BFloat16)). For |
| 183 | + // pattern b the value is Float.intBitsToFloat(b << 16): non-NaN values (incl. +/-0, |
| 184 | + // subnormals, +/-Infinity) must return bit-for-bit, while NaN inputs collapse to a single |
| 185 | + // canonical NaN on write and are only required to read back as NaN. Truncation of |
| 186 | + // non-representable float inputs is covered by testBFloat16TruncationMatchesServer. |
| 187 | + final String table = "test_bfloat16"; |
| 188 | + final int count = 1 << 16; |
| 189 | + List<DTOForBFloat16Tests> data = new ArrayList<>(count); |
| 190 | + for (int b = 0; b < count; b++) { |
| 191 | + float value = Float.intBitsToFloat(b << 16); |
| 192 | + // Row 0 exercises the null path; every other row round-trips a non-null Nullable(BFloat16). |
| 193 | + data.add(new DTOForBFloat16Tests(b, value, b == 0 ? null : value)); |
| 194 | + } |
| 195 | + |
| 196 | + writeReadVerify(table, |
| 197 | + DTOForBFloat16Tests.tblCreateSQL(table), |
| 198 | + DTOForBFloat16Tests.class, |
| 199 | + data, |
| 200 | + (all, dto) -> { |
| 201 | + DTOForBFloat16Tests expected = all.get(dto.getRowId()); |
| 202 | + assertBFloat16Equals(dto.getBFloat16(), expected.getBFloat16()); |
| 203 | + assertBFloat16Equals(dto.getBFloat16Nullable(), expected.getBFloat16Nullable()); |
| 204 | + }); |
| 205 | + } |
| 206 | + |
| 207 | + // BFloat16 keeps the high 16 bits of a float32, so every non-NaN value round-trips bit-for-bit; |
| 208 | + // NaN inputs collapse to a single canonical NaN on write and are only required to read back as NaN. |
| 209 | + private static void assertBFloat16Equals(Float actual, Float expected) { |
| 210 | + if (expected == null) { |
| 211 | + Assert.assertNull(actual); |
| 212 | + } else if (Float.isNaN(expected)) { |
| 213 | + Assert.assertNotNull(actual); |
| 214 | + Assert.assertTrue(Float.isNaN(actual), "expected a NaN but got " + actual); |
| 215 | + } else { |
| 216 | + Assert.assertNotNull(actual); |
| 217 | + Assert.assertEquals(Float.floatToRawIntBits(actual), Float.floatToRawIntBits(expected)); |
| 218 | + } |
| 219 | + } |
| 220 | + |
| 221 | + @Test(groups = {"integration"}) |
| 222 | + public void testBFloat16ReadFromServerWrittenValues() throws Exception { |
| 223 | + if (isVersionMatch(BFLOAT16_UNSUPPORTED_VERSIONS)) { |
| 224 | + throw new SkipException("BFloat16 requires ClickHouse 24.11+"); |
| 225 | + } |
| 226 | + |
| 227 | + // Exhaustively cover reading every one of the 2^16 BFloat16 bit patterns. The dataset is |
| 228 | + // written with SQL text literals so the *server* produces the stored bytes; this decouples |
| 229 | + // the read path under test from the client's binary write path. Row b holds the BFloat16 |
| 230 | + // whose bit pattern is b, so a read must widen it to Float.intBitsToFloat(b << 16) (NaN |
| 231 | + // inputs collapse to a single canonical NaN on the server and only read back as NaN). |
| 232 | + final String table = "test_bfloat16_read"; |
| 233 | + final int count = 1 << 16; |
| 234 | + final int batchSize = 4096; // keep each INSERT well under max_query_size |
| 235 | + client.execute("DROP TABLE IF EXISTS " + table).get(); |
| 236 | + client.execute("CREATE TABLE " + table |
| 237 | + + " (rowId Int32, v BFloat16, vNull Nullable(BFloat16)) ENGINE = MergeTree ORDER BY rowId").get(); |
| 238 | + |
| 239 | + for (int start = 0; start < count; start += batchSize) { |
| 240 | + int end = Math.min(start + batchSize, count); |
| 241 | + StringBuilder insert = new StringBuilder("INSERT INTO ").append(table).append(" VALUES "); |
| 242 | + for (int b = start; b < end; b++) { |
| 243 | + String literal = bFloat16Literal(b); |
| 244 | + if (b > start) { |
| 245 | + insert.append(','); |
| 246 | + } |
| 247 | + insert.append('(').append(b).append(',').append(literal).append(',') |
| 248 | + .append(b == 0 ? "NULL" : literal).append(')'); |
| 249 | + } |
| 250 | + client.execute(insert.toString()).get(); |
| 251 | + } |
| 252 | + |
| 253 | + // The server's stored bits are the ground truth for the read: reinterpretAsUInt16(v) yields |
| 254 | + // the exact 16-bit pattern the server wrote, so a correct read must widen it to |
| 255 | + // Float.intBitsToFloat(bits << 16). Deriving the expectation from the stored bits (rather than |
| 256 | + // from rowId) keeps the read assertion correct regardless of how the server converts the text |
| 257 | + // literal to BFloat16 - in particular the server flushes BFloat16 subnormals to zero, so those |
| 258 | + // patterns are stored as +/-0 even though the literal named a tiny subnormal. |
| 259 | + AtomicInteger rowCount = new AtomicInteger(0); |
| 260 | + client.queryAll("SELECT rowId, v, reinterpretAsUInt16(v) AS bits, vNull FROM " + table + " ORDER BY rowId") |
| 261 | + .forEach(row -> { |
| 262 | + int b = row.getInteger("rowId"); |
| 263 | + float expected = Float.intBitsToFloat(row.getInteger("bits") << 16); |
| 264 | + assertBFloat16Equals(row.getFloat("v"), expected); |
| 265 | + if (b == 0) { |
| 266 | + Assert.assertNull(row.getObject("vNull")); |
| 267 | + } else { |
| 268 | + assertBFloat16Equals((Float) row.getObject("vNull"), expected); |
| 269 | + } |
| 270 | + rowCount.incrementAndGet(); |
| 271 | + }); |
| 272 | + Assert.assertEquals(rowCount.get(), count); |
| 273 | + } |
| 274 | + |
| 275 | + // Text form of the exact BFloat16 whose bit pattern is {@code pattern}, suitable for use as a |
| 276 | + // SQL numeric literal. Non-finite patterns use ClickHouse's nan/inf/-inf tokens; every other |
| 277 | + // pattern uses the shortest round-trippable decimal of Float.intBitsToFloat(pattern << 16), |
| 278 | + // which is exactly representable in BFloat16 (its low 16 mantissa bits are zero). |
| 279 | + private static String bFloat16Literal(int pattern) { |
| 280 | + float value = Float.intBitsToFloat(pattern << 16); |
| 281 | + if (Float.isNaN(value)) { |
| 282 | + return "nan"; |
| 283 | + } |
| 284 | + if (value == Float.POSITIVE_INFINITY) { |
| 285 | + return "inf"; |
| 286 | + } |
| 287 | + if (value == Float.NEGATIVE_INFINITY) { |
| 288 | + return "-inf"; |
| 289 | + } |
| 290 | + return Float.toString(value); |
| 291 | + } |
| 292 | + |
| 293 | + @Test(groups = {"integration"}) |
| 294 | + public void testBFloat16TruncationMatchesServer() throws Exception { |
| 295 | + if (isVersionMatch(BFLOAT16_UNSUPPORTED_VERSIONS)) { |
| 296 | + throw new SkipException("BFloat16 requires ClickHouse 24.11+"); |
| 297 | + } |
| 298 | + |
| 299 | + // (a) The client must decode a BFloat16 produced by the server (Float32 -> BFloat16 |
| 300 | + // drops the low mantissa bits) to the same value. |
| 301 | + List<GenericRecord> cast = client.queryAll( |
| 302 | + "SELECT CAST(3.14 AS BFloat16) AS v, CAST(0.1 AS BFloat16) AS v2"); |
| 303 | + Assert.assertEquals(cast.get(0).getFloat("v"), 3.125f, 0.0f); // 0x4048F5C3 -> 0x40480000 |
| 304 | + Assert.assertEquals(cast.get(0).getFloat("v2"), 0.099609375f, 0.0f); // 0x3DCCCCCD -> 0x3DCC0000 |
| 305 | + |
| 306 | + // (b) A value written by the client must be stored byte-for-byte identically to the |
| 307 | + // server's own Float32 -> BFloat16 conversion of the same value. |
| 308 | + final String table = "test_bfloat16_truncation"; |
| 309 | + writeReadVerify(table, |
| 310 | + DTOForBFloat16Tests.tblCreateSQL(table), |
| 311 | + DTOForBFloat16Tests.class, |
| 312 | + Arrays.asList(new DTOForBFloat16Tests(0, 3.14f, 0.1f)), |
| 313 | + (data, dto) -> { |
| 314 | + Assert.assertEquals(dto.getBFloat16(), 3.125f, 0.0f); |
| 315 | + Assert.assertEquals(dto.getBFloat16Nullable(), Float.valueOf(0.099609375f)); |
| 316 | + }); |
| 317 | + List<GenericRecord> parity = client.queryAll( |
| 318 | + "SELECT reinterpretAsUInt16(bFloat16) AS written, " + |
| 319 | + "reinterpretAsUInt16(CAST(toFloat32(3.14) AS BFloat16)) AS server FROM " + table); |
| 320 | + Assert.assertEquals(parity.get(0).getInteger("written"), parity.get(0).getInteger("server")); |
| 321 | + } |
| 322 | + |
| 323 | + @Test(groups = {"integration"}) |
| 324 | + public void testBFloat16InDynamicColumn() throws Exception { |
| 325 | + if (isVersionMatch("(,24.8]") || isVersionMatch(BFLOAT16_UNSUPPORTED_VERSIONS)) { |
| 326 | + throw new SkipException("BFloat16 requires ClickHouse 24.11+"); |
| 327 | + } |
| 328 | + |
| 329 | + // Reading a BFloat16 held in a Dynamic column exercises the dynamic type-tag read path. |
| 330 | + List<GenericRecord> rows = client.queryAll( |
| 331 | + "SELECT CAST(CAST(1.5 AS BFloat16) AS Dynamic) AS v SETTINGS allow_experimental_dynamic_type = 1"); |
| 332 | + Assert.assertEquals(rows.get(0).getObject("v"), Float.valueOf(1.5f)); |
| 333 | + } |
| 334 | + |
157 | 335 | @Test(groups = {"integration"}) |
158 | 336 | public void testVariantWithSimpleDataTypes() throws Exception { |
159 | 337 | if (isVersionMatch("(,24.8]")) { |
|
0 commit comments