Skip to content

Commit 3e8599b

Browse files
wjxiz1992wjxiz1992
committed
Split Java ORC calendar bindings into a follow-up PR
Signed-off-by: Allen Xu <allxu@nvidia.com>
2 parents 82fec42 + 13be0e2 commit 3e8599b

10 files changed

Lines changed: 218 additions & 466 deletions

File tree

‎cpp/src/io/csv/csv_gpu.cu‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@ CUDF_KERNEL void __launch_bounds__(csvparse_block_dim)
192192
field_start = cudf::io::gpu::skip_leading_delimiter_run(field_start, row_end, opts);
193193
if (field_start >= row_end) break;
194194
auto next_delimiter = cudf::io::gpu::seek_field_end(field_start, row_end, opts);
195+
// Trailing whitespace before the record terminator does not start another field
196+
if (opts.multi_delimiter and next_delimiter == field_start) { break; }
195197

196198
// Checking if this is a column that the user wants --- user can filter columns
197199
if (column_flags[col] & column_parse::inferred) {
@@ -288,6 +290,10 @@ CUDF_KERNEL void __launch_bounds__(csvparse_block_dim)
288290
}
289291
actual_col++;
290292
}
293+
// Row offsets can span skipped comments or blank lines. Only a delimiter
294+
// starts another field; a record terminator must leave missing fields null.
295+
// TODO: Trim row_end at the record terminator instead (#24642).
296+
if (not cudf::io::gpu::is_field_delimiter(next_delimiter, row_end, opts)) { break; }
291297
next_field = next_delimiter + 1;
292298
field_start = next_field;
293299
col++;
@@ -345,6 +351,8 @@ CUDF_KERNEL void __launch_bounds__(csvparse_block_dim)
345351
if (field_start >= row_end) break;
346352
next_field = field_start;
347353
auto next_delimiter = cudf::io::gpu::seek_field_end(field_start, row_end, options);
354+
// Trailing whitespace before the record terminator does not start another field
355+
if (options.multi_delimiter and next_delimiter == field_start) { break; }
348356

349357
if (column_flags[col] & column_parse::enabled) {
350358
// check if the entire field is a NaN string - consistent with pandas
@@ -416,6 +424,10 @@ CUDF_KERNEL void __launch_bounds__(csvparse_block_dim)
416424
}
417425
++actual_col;
418426
}
427+
// Row offsets can span skipped comments or blank lines. Only a delimiter
428+
// starts another field; a record terminator must leave missing fields null.
429+
// TODO: Trim row_end at the record terminator instead (#24642).
430+
if (not cudf::io::gpu::is_field_delimiter(next_delimiter, row_end, options)) { break; }
419431
next_field = next_delimiter + 1;
420432
field_start = next_field;
421433
++col;

‎cpp/src/io/csv/reader_impl.cu‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,8 @@ void infer_column_types(parse_options const& parse_opts,
640640
for (auto col_idx = 0u; col_idx < column_flags.size(); ++col_idx) {
641641
if (not(column_flags[col_idx] & column_parse::inferred)) { continue; }
642642
auto const& stats = column_stats[inf_col_idx++];
643-
if (stats.null_count == num_records or stats.total_count() == 0) {
643+
// Missing fields are absent from the histogram and also represent nulls.
644+
if (stats.null_count == stats.total_count()) {
644645
// Entire column is NULL; allocate the smallest amount of memory
645646
column_types[col_idx] = data_type(cudf::type_id::INT8);
646647
} else if (stats.string_count > 0L) {

‎cpp/src/io/utilities/parsing_utils.cuh‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,9 @@ __device__ __inline__ char const* skip_leading_delimiter_run(char const* begin,
202202
}
203203

204204
/**
205-
* @brief CUDA kernel iterates over the data until the end of the current field
205+
* @brief Iterate over the data until the end of the current field.
206206
*
207-
* Also iterates over (one or more) delimiter characters after the field.
207+
* Delimiter runs are skipped by the caller before scanning the next field.
208208
* Function applies to formats with field delimiters and line terminators.
209209
*
210210
* @param begin Pointer to the first element of the string
@@ -213,8 +213,7 @@ __device__ __inline__ char const* skip_leading_delimiter_run(char const* begin,
213213
* @param escape_char A boolean value to signify whether to consider `\` as escape character or
214214
* just a character.
215215
*
216-
* @return Pointer to the last character in the field, including the
217-
* delimiter(s) following the field data
216+
* @return Pointer to the first delimiter or record terminator after the field, or `end`.
218217
*/
219218
__device__ __inline__ char const* seek_field_end(char const* begin,
220219
char const* end,
@@ -237,9 +236,6 @@ __device__ __inline__ char const* seek_field_end(char const* begin,
237236
quotation = !quotation;
238237
} else if (!quotation) {
239238
if (*current == opts.delimiter) {
240-
while (opts.multi_delimiter && (current + 1 < end) && *(current + 1) == opts.delimiter) {
241-
++current;
242-
}
243239
break;
244240
} else if (*current == opts.terminator) {
245241
break;
@@ -263,6 +259,16 @@ __device__ __inline__ char const* seek_field_end(char const* begin,
263259
return current;
264260
}
265261

262+
/**
263+
* @brief Returns whether the field scan stopped at a delimiter within the record.
264+
*/
265+
__device__ __inline__ bool is_field_delimiter(char const* field_end,
266+
char const* row_end,
267+
parse_options_view const& opts)
268+
{
269+
return field_end < row_end and *field_end == opts.delimiter;
270+
}
271+
266272
/**
267273
* @brief Lexicographically compare digits in input against string
268274
* representing an integer

‎cpp/tests/io/csv_test.cpp‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <cudf_test/base_fixture.hpp>
77
#include <cudf_test/column_utilities.hpp>
88
#include <cudf_test/column_wrapper.hpp>
9+
#include <cudf_test/cudf_gtest.hpp>
910
#include <cudf_test/default_stream.hpp>
1011
#include <cudf_test/iterator_utilities.hpp>
1112
#include <cudf_test/memory_resource_utilities.hpp>
@@ -1441,6 +1442,170 @@ TEST_F(CsvReaderTest, BlanksAndComments)
14411442
expect_column_data_equal(std::vector<int32_t>{1, 3, 4, 5, 8, 9}, view.column(0));
14421443
}
14431444

1445+
struct CsvShortRecordTest : public CsvReaderTest,
1446+
public testing::WithParamInterface<std::tuple<std::string, bool>> {
1447+
auto read(std::string const& buffer)
1448+
{
1449+
auto const& [separator, infer_types] = GetParam();
1450+
auto builder = cudf::io::csv_reader_options::builder(
1451+
cudf::io::source_info{cudf::host_span<std::byte const>{
1452+
reinterpret_cast<std::byte const*>(buffer.data()), buffer.size()}})
1453+
.header(0)
1454+
.comment('#')
1455+
.lineterminator(separator.back())
1456+
.keep_default_na(false);
1457+
if (!infer_types) { builder.dtypes(std::vector<data_type>(3, dtype<cudf::string_view>())); }
1458+
return cudf::io::read_csv(builder.build());
1459+
}
1460+
1461+
void expect_short_rows(std::string const& skipped)
1462+
{
1463+
auto const& [separator, infer_types] = GetParam();
1464+
std::string const buffer = "a,b,c" + separator + "1" + separator + skipped + "2,3,4" +
1465+
separator + "5" + separator + skipped + "6";
1466+
auto const result = read(buffer);
1467+
if (infer_types) {
1468+
cudf::test::fixed_width_column_wrapper<int64_t> const a{1, 2, 5, 6};
1469+
cudf::test::fixed_width_column_wrapper<int64_t> const b{{0, 3, 0, 0},
1470+
{false, true, false, false}};
1471+
cudf::test::fixed_width_column_wrapper<int64_t> const c{{0, 4, 0, 0},
1472+
{false, true, false, false}};
1473+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1474+
} else {
1475+
cudf::test::strings_column_wrapper const a{"1", "2", "5", "6"};
1476+
cudf::test::strings_column_wrapper const b{{"", "3", "", ""}, {false, true, false, false}};
1477+
cudf::test::strings_column_wrapper const c{{"", "4", "", ""}, {false, true, false, false}};
1478+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1479+
}
1480+
}
1481+
};
1482+
1483+
INSTANTIATE_TEST_SUITE_P(
1484+
CsvShortRecords,
1485+
CsvShortRecordTest,
1486+
testing::Combine(testing::Values(std::string{"\n"}, std::string{"\r\n"}, std::string{";"}),
1487+
testing::Bool()),
1488+
[](auto const& info) {
1489+
auto const& separator = std::get<0>(info.param);
1490+
return std::string{separator == "\n" ? "Lf"
1491+
: separator == "\r\n" ? "CrLf"
1492+
: "Custom"} +
1493+
(std::get<1>(info.param) ? "Inferred" : "Strings");
1494+
});
1495+
1496+
TEST_P(CsvShortRecordTest, MissingFieldsBeforeComments)
1497+
{
1498+
auto const& separator = std::get<0>(GetParam());
1499+
expect_short_rows("#not,a,column" + separator + "#another,ignored,line" + separator);
1500+
}
1501+
1502+
TEST_P(CsvShortRecordTest, MissingFieldsBeforeBlanks)
1503+
{
1504+
auto const& separator = std::get<0>(GetParam());
1505+
expect_short_rows(separator + separator);
1506+
}
1507+
1508+
TEST_P(CsvShortRecordTest, QuotedTerminatorsAndComments)
1509+
{
1510+
auto const& [separator, infer_types] = GetParam();
1511+
auto const multiline = "line" + separator + "#é,inside";
1512+
std::string const buffer = "a,b,c" + separator + "1,\"" + multiline + "\",7" + separator +
1513+
"#ignored,comment,99" + separator + "2,\"#é,quoted\"" + separator +
1514+
"#skipped,fields,123" + separator + "3,tail,8";
1515+
auto const result = read(buffer);
1516+
cudf::test::strings_column_wrapper const b{multiline, "#é,quoted", "tail"};
1517+
if (infer_types) {
1518+
cudf::test::fixed_width_column_wrapper<int64_t> const a{1, 2, 3};
1519+
cudf::test::fixed_width_column_wrapper<int64_t> const c{{7, 0, 8}, {true, false, true}};
1520+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1521+
} else {
1522+
cudf::test::strings_column_wrapper const a{"1", "2", "3"};
1523+
cudf::test::strings_column_wrapper const c{{"7", "", "8"}, {true, false, true}};
1524+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1525+
}
1526+
}
1527+
1528+
TEST_F(CsvReaderTest, ShortRecordBeforeCommentDelimWhitespace)
1529+
{
1530+
for (auto const& trailing_space : {std::string{""}, std::string{" "}, std::string{" "}}) {
1531+
for (bool infer_types : {false, true}) {
1532+
SCOPED_TRACE(trailing_space);
1533+
SCOPED_TRACE(infer_types);
1534+
std::string const buffer = "a b c\n1" + trailing_space + "\n#x y z\n2 3 4\n";
1535+
auto builder = cudf::io::csv_reader_options::builder(
1536+
cudf::io::source_info{cudf::host_span<std::byte const>{
1537+
reinterpret_cast<std::byte const*>(buffer.data()), buffer.size()}})
1538+
.delim_whitespace(true)
1539+
.comment('#')
1540+
.keep_default_na(false);
1541+
if (!infer_types) { builder.dtypes(std::vector<data_type>(3, dtype<cudf::string_view>())); }
1542+
auto const result = cudf::io::read_csv(builder.build());
1543+
1544+
if (infer_types) {
1545+
cudf::test::fixed_width_column_wrapper<int64_t> const a{1, 2};
1546+
cudf::test::fixed_width_column_wrapper<int64_t> const b{{0, 3}, {false, true}};
1547+
cudf::test::fixed_width_column_wrapper<int64_t> const c{{0, 4}, {false, true}};
1548+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1549+
} else {
1550+
cudf::test::strings_column_wrapper const a{"1", "2"};
1551+
cudf::test::strings_column_wrapper const b({"", "3"}, {false, true});
1552+
cudf::test::strings_column_wrapper const c({"", "4"}, {false, true});
1553+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1554+
}
1555+
}
1556+
}
1557+
}
1558+
1559+
TEST_F(CsvReaderTest, ShortRecordAllNullColumn)
1560+
{
1561+
// Missing fields and explicit NA tokens must infer the same all-null type.
1562+
for (auto const& middle_row : {std::string{"2"}, std::string{"2,NA"}}) {
1563+
SCOPED_TRACE(middle_row);
1564+
std::string const buffer = "a,b\n1,NA\n" + middle_row + "\n#ignored\n3,NA\n";
1565+
auto const result =
1566+
cudf::io::read_csv(cudf::io::csv_reader_options::builder(
1567+
cudf::io::source_info{cudf::host_span<std::byte const>{
1568+
reinterpret_cast<std::byte const*>(buffer.data()), buffer.size()}})
1569+
.comment('#'));
1570+
cudf::test::fixed_width_column_wrapper<int64_t> const a{1, 2, 3};
1571+
cudf::test::fixed_width_column_wrapper<int8_t> const b{{0, 0, 0},
1572+
cudf::test::iterators::all_nulls()};
1573+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b}}, result.tbl->view());
1574+
}
1575+
}
1576+
1577+
TEST_F(CsvReaderTest, ShortRecordExplicitEmptyField)
1578+
{
1579+
std::string const buffer = "a,b,c\n1,\n#ignored,comment,fields\n2,3,4\n";
1580+
auto const result =
1581+
cudf::io::read_csv(cudf::io::csv_reader_options::builder(
1582+
cudf::io::source_info{cudf::host_span<std::byte const>{
1583+
reinterpret_cast<std::byte const*>(buffer.data()), buffer.size()}})
1584+
.comment('#')
1585+
.keep_default_na(false)
1586+
.dtypes(std::vector<data_type>(3, dtype<cudf::string_view>())));
1587+
cudf::test::strings_column_wrapper const a{"1", "2"};
1588+
cudf::test::strings_column_wrapper const b{"", "3"};
1589+
cudf::test::strings_column_wrapper const c({"", "4"}, {false, true});
1590+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, b, c}}, result.tbl->view());
1591+
}
1592+
1593+
TEST_F(CsvReaderTest, ShortRecordBeforeCommentUseCols)
1594+
{
1595+
std::string const buffer = "a,b,c\n1\n#x,y,z\n2,3,4\n";
1596+
auto const in_opts = cudf::io::csv_reader_options::builder(
1597+
cudf::io::source_info{cudf::host_span<std::byte const>{
1598+
reinterpret_cast<std::byte const*>(buffer.data()), buffer.size()}})
1599+
.comment('#')
1600+
.use_cols_names({"a", "c"})
1601+
.build();
1602+
auto const result = cudf::io::read_csv(in_opts);
1603+
1604+
cudf::test::fixed_width_column_wrapper<int64_t> const a{1, 2};
1605+
cudf::test::fixed_width_column_wrapper<int64_t> const c{{0, 4}, {false, true}};
1606+
CUDF_TEST_EXPECT_TABLES_EQUIVALENT(table_view{{a, c}}, result.tbl->view());
1607+
}
1608+
14441609
TEST_F(CsvReaderTest, EmptyFile)
14451610
{
14461611
auto filepath = temp_env->get_temp_dir() + "EmptyFile.csv";

‎java/pom.xml‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -136,32 +136,6 @@
136136
</pluginRepositories>
137137

138138
<dependencies>
139-
<dependency>
140-
<groupId>org.apache.orc</groupId>
141-
<artifactId>orc-core</artifactId>
142-
<version>1.9.4</version>
143-
<classifier>shaded-protobuf</classifier>
144-
<scope>test</scope>
145-
<exclusions>
146-
<!-- Use the hadoop-common dependency below instead of duplicate Hadoop classes. -->
147-
<exclusion>
148-
<groupId>org.apache.hadoop</groupId>
149-
<artifactId>hadoop-client-api</artifactId>
150-
</exclusion>
151-
</exclusions>
152-
</dependency>
153-
<dependency>
154-
<groupId>org.apache.hadoop</groupId>
155-
<artifactId>hadoop-hdfs-client</artifactId>
156-
<version>3.4.0</version>
157-
<scope>test</scope>
158-
</dependency>
159-
<dependency>
160-
<groupId>org.apache.hive</groupId>
161-
<artifactId>hive-storage-api</artifactId>
162-
<version>2.8.1</version>
163-
<scope>test</scope>
164-
</dependency>
165139
<dependency>
166140
<groupId>org.slf4j</groupId>
167141
<artifactId>slf4j-api</artifactId>

‎java/src/main/java/ai/rapids/cudf/ORCOptions.java‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
*
3-
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
* SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION.
44
* SPDX-License-Identifier: Apache-2.0
55
*
66
*/
@@ -19,7 +19,6 @@ public class ORCOptions extends ColumnFilterOptions {
1919
public static ORCOptions DEFAULT = new ORCOptions(new Builder());
2020

2121
private final boolean useNumPyTypes;
22-
private final boolean convertDatesToProleptic;
2322
private final DType unit;
2423
private final String[] decimal128Columns;
2524
private final boolean ignoreTimezoneInStripeFooter;
@@ -28,7 +27,6 @@ private ORCOptions(Builder builder) {
2827
super(builder);
2928
decimal128Columns = builder.decimal128Columns.toArray(new String[0]);
3029
useNumPyTypes = builder.useNumPyTypes;
31-
convertDatesToProleptic = builder.convertDatesToProleptic;
3230
unit = builder.unit;
3331
ignoreTimezoneInStripeFooter = builder.ignoreTimezoneInStripeFooter;
3432
}
@@ -37,8 +35,6 @@ boolean usingNumPyTypes() {
3735
return useNumPyTypes;
3836
}
3937

40-
boolean convertDatesToProleptic() { return convertDatesToProleptic; }
41-
4238
DType timeUnit() {
4339
return unit;
4440
}
@@ -57,7 +53,6 @@ public static Builder builder() {
5753

5854
public static class Builder extends ColumnFilterOptions.Builder<Builder> {
5955
private boolean useNumPyTypes = true;
60-
private boolean convertDatesToProleptic = false;
6156
private DType unit = DType.EMPTY;
6257

6358
final List<String> decimal128Columns = new ArrayList<>();
@@ -104,19 +99,6 @@ public Builder decimal128Column(String... names) {
10499
return this;
105100
}
106101

107-
/**
108-
* Convert DATE columns explicitly marked JULIAN_GREGORIAN to cuDF proleptic dates.
109-
* Disabled by default for callers that perform their own rebasing. Missing and UNKNOWN
110-
* calendar fields and TIMESTAMP columns are unchanged. Conversion applies to nested DATE
111-
* columns and precedes NumPy promotion. Unrepresentable dates cause a read error.
112-
* @param convert whether DATE conversion is enabled
113-
* @return builder for chaining
114-
*/
115-
public Builder convertDatesToProleptic(boolean convert) {
116-
this.convertDatesToProleptic = convert;
117-
return this;
118-
}
119-
120102
public Builder ignoreTimezoneInStripeFooter() {
121103
this.ignoreTimezoneInStripeFooter = true;
122104
return this;

0 commit comments

Comments
 (0)