Skip to content
22 changes: 21 additions & 1 deletion cpp/include/cudf/io/orc.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -277,9 +277,26 @@ class orc_reader_options {
/**
* @brief Sets timestamp type to which timestamp column will be cast.
*
* Only TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, TIMESTAMP_MICROSECONDS and
* TIMESTAMP_NANOSECONDS are supported as the target type; EMPTY, the default, reads timestamps
* as TIMESTAMP_NANOSECONDS. For a coarser resolution such as TIMESTAMP_DAYS, cast the output
* column instead.
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
*
* @param type Type of timestamp
*
* @throw cudf::logic_error if `type` is not EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS,
* TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
*/
void set_timestamp_type(data_type type) { _timestamp_type = type; }
void set_timestamp_type(data_type type)
{
CUDF_EXPECTS(type.id() == type_id::EMPTY or type.id() == type_id::TIMESTAMP_SECONDS or
type.id() == type_id::TIMESTAMP_MILLISECONDS or
type.id() == type_id::TIMESTAMP_MICROSECONDS or
type.id() == type_id::TIMESTAMP_NANOSECONDS,
"timestamp_type must be EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, "
"TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS");
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
_timestamp_type = type;
}

/**
* @brief Set columns that should be read as 128-bit Decimal
Expand Down Expand Up @@ -405,6 +422,9 @@ class orc_reader_options_builder {
*
* @param type Type of timestamp
* @return this for chaining
*
* @throw cudf::logic_error if `type` is not EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS,
* TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS
*/
orc_reader_options_builder& timestamp_type(data_type type)
{
Expand Down
22 changes: 21 additions & 1 deletion cpp/include/cudf/io/parquet.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -619,9 +619,26 @@ class parquet_reader_options {
/**
* @brief Sets timestamp_type used to cast timestamp columns.
*
* Only TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, TIMESTAMP_MICROSECONDS and
* TIMESTAMP_NANOSECONDS are supported as the target type; EMPTY, the default, keeps each
* column's own resolution. For a coarser resolution such as TIMESTAMP_DAYS, cast the output
* column instead.
*
* @throws cudf::logic_error if `type` is not EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS,
* TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS
*
* @param type The timestamp data_type to which all timestamp columns need to be cast
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
*/
void set_timestamp_type(data_type type) { _timestamp_type = type; }
void set_timestamp_type(data_type type)
{
CUDF_EXPECTS(type.id() == type_id::EMPTY or type.id() == type_id::TIMESTAMP_SECONDS or
type.id() == type_id::TIMESTAMP_MILLISECONDS or
type.id() == type_id::TIMESTAMP_MICROSECONDS or
type.id() == type_id::TIMESTAMP_NANOSECONDS,
"timestamp_type must be EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, "
"TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS");
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
_timestamp_type = type;
}

/**
* @brief Sets decimal width used to cast decimal columns.
Expand Down Expand Up @@ -900,6 +917,9 @@ class parquet_reader_options_builder {
/**
* @brief timestamp_type used to cast timestamp columns.
*
* @throws cudf::logic_error if `type` is not EMPTY, TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS,
* TIMESTAMP_MICROSECONDS or TIMESTAMP_NANOSECONDS
*
* @param type The timestamp data_type to which all timestamp columns need to be cast
* @return this for chaining
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
*/
Expand Down
25 changes: 25 additions & 0 deletions cpp/tests/io/orc_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1539,6 +1539,31 @@ TEST_F(OrcReaderTest, ZeroColumnsPreservesRowCount)
EXPECT_EQ(result.tbl->view().num_rows(), num_rows);
}

TEST_F(OrcReaderTest, TimestampTypeMustHaveSecondBasedUnit)
{
// The decode kernel converts timestamps to seconds, milliseconds, microseconds or nanoseconds
// and stores 64-bit values, so a request for TIMESTAMP_DAYS would write 8-byte nanosecond
// counts into a column allocated at 4 bytes per row.
auto options = cudf::io::orc_reader_options{};
for (auto const id : {cudf::type_id::EMPTY,
cudf::type_id::TIMESTAMP_SECONDS,
cudf::type_id::TIMESTAMP_MILLISECONDS,
cudf::type_id::TIMESTAMP_MICROSECONDS,
cudf::type_id::TIMESTAMP_NANOSECONDS}) {
EXPECT_NO_THROW(options.set_timestamp_type(cudf::data_type{id}));
EXPECT_EQ(options.get_timestamp_type().id(), id);
}
for (auto const id :
{cudf::type_id::TIMESTAMP_DAYS, cudf::type_id::DURATION_SECONDS, cudf::type_id::INT64}) {
EXPECT_THROW(options.set_timestamp_type(cudf::data_type{id}), cudf::logic_error);
}
// A rejected type leaves the previous one in place.
EXPECT_EQ(options.get_timestamp_type().id(), cudf::type_id::TIMESTAMP_NANOSECONDS);
EXPECT_THROW(cudf::io::orc_reader_options_builder{}.timestamp_type(
cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}),
cudf::logic_error);
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
}

TEST_F(OrcReaderTest, SingleInputs)
{
srand(31533);
Expand Down
25 changes: 25 additions & 0 deletions cpp/tests/io/parquet_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,31 @@ TEST_F(ParquetReaderTest, SelectColumnByMissingFieldIds)
}
}

TEST_F(ParquetReaderTest, TimestampTypeMustHaveSecondBasedUnit)
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
{
// The reader rescales timestamps by a clock rate in ticks per second, which TIMESTAMP_DAYS
// cannot express: a request for it used to decode seconds as 8-byte values into a column
// allocated at 4 bytes per row.
auto options = cudf::io::parquet_reader_options{};
for (auto const id : {cudf::type_id::EMPTY,
cudf::type_id::TIMESTAMP_SECONDS,
cudf::type_id::TIMESTAMP_MILLISECONDS,
cudf::type_id::TIMESTAMP_MICROSECONDS,
cudf::type_id::TIMESTAMP_NANOSECONDS}) {
EXPECT_NO_THROW(options.set_timestamp_type(cudf::data_type{id}));
EXPECT_EQ(options.get_timestamp_type().id(), id);
}
for (auto const id :
{cudf::type_id::TIMESTAMP_DAYS, cudf::type_id::DURATION_SECONDS, cudf::type_id::INT64}) {
EXPECT_THROW(options.set_timestamp_type(cudf::data_type{id}), cudf::logic_error);
}
// A rejected type leaves the previous one in place.
EXPECT_EQ(options.get_timestamp_type().id(), cudf::type_id::TIMESTAMP_NANOSECONDS);
EXPECT_THROW(cudf::io::parquet_reader_options_builder{}.timestamp_type(
cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}),
cudf::logic_error);
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
}

TEST_F(ParquetReaderTest, ColumnSelectionModesAreExclusive)
{
auto selected_by_name = cudf::io::parquet_reader_options{};
Expand Down
6 changes: 4 additions & 2 deletions java/src/main/java/ai/rapids/cudf/ORCOptions.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
*
* SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
*/
Expand Down Expand Up @@ -73,11 +73,13 @@ public Builder withNumPyTypes(boolean useNumPyTypes) {

/**
* Specify the time unit to use when returning timestamps.
* Only TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, TIMESTAMP_MICROSECONDS and
* TIMESTAMP_NANOSECONDS are valid; for TIMESTAMP_DAYS, cast the returned column instead.
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
* @param unit default unit of time specified by the user
* @return builder for chaining
*/
public ORCOptions.Builder withTimeUnit(DType unit) {
assert unit.isTimestampType();
assert unit == DType.EMPTY || unit.hasTimeResolution();
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
this.unit = unit;
return this;
}
Expand Down
6 changes: 4 additions & 2 deletions java/src/main/java/ai/rapids/cudf/ParquetOptions.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
*
* SPDX-FileCopyrightText: Copyright (c) 2019, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
*/
Expand Down Expand Up @@ -48,11 +48,13 @@ public static class Builder extends ColumnFilterOptions.Builder<Builder> {

/**
* Specify the time unit to use when returning timestamps.
* Only TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS, TIMESTAMP_MICROSECONDS and
* TIMESTAMP_NANOSECONDS are valid; for TIMESTAMP_DAYS, cast the returned column instead.
* @param unit default unit of time specified by the user
* @return builder for chaining
*/
public Builder withTimeUnit(DType unit) {
assert unit.isTimestampType();
assert unit == DType.EMPTY || unit.hasTimeResolution();
Comment thread
galipremsagar marked this conversation as resolved.
Resolved
this.unit = unit;
return this;
}
Expand Down
Loading