Skip to content

ORC-2222: [C++] Support Java fixed-offset timezone IDs #2716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ffacs wants to merge 1 commit into from
Closed

ORC-2222: [C++] Support Java fixed-offset timezone IDs #2716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions c++/src/Timezone.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,65 @@ namespace orc {
return buffer;
}

// Java TimeZone IDs use GMT followed by an ISO-style offset, not the
// reversed sign used by POSIX TZ strings. ZoneOffset can also include seconds.
static bool parseFixedOffset(const std::string& zone, int64_t& offset) {
if ((zone.size() != 9 && zone.size() != 12) || zone.compare(0, 3, "GMT") != 0 ||
(zone[3] != '+' && zone[3] != '-')) {
return false;
}
int64_t parts[3] = {0, 0, 0};
for (size_t i = 0; i < (zone.size() == 9 ? 2 : 3); ++i) {
size_t start = 4 + i * 3;
if ((i != 0 && zone[start - 1] != ':') || zone[start] < '0' || zone[start] > '9' ||
zone[start + 1] < '0' || zone[start + 1] > '9') {
return false;
}
parts[i] = (zone[start] - '0') * 10 + zone[start + 1] - '0';
}
if (parts[0] > 23 || parts[1] > 59 || parts[2] > 59) {
return false;
}
offset = (parts[0] * 60 + parts[1]) * 60 + parts[2];
if (zone[3] == '-') {
offset = -offset;
}
return true;
}

class FixedOffsetTimezone : public Timezone {
public:
FixedOffsetTimezone(const std::string& name, int64_t offset) : variant_{offset, false, name} {}

const TimezoneVariant& getVariant(int64_t) const override {
return variant_;
}

int64_t getEpoch() const override {
// 2015-01-01 00:00:00 in this timezone, expressed as Unix seconds.
return 1420070400 - variant_.gmtOffset;
}

void print(std::ostream& out) const override {
out << variant_.toString() << "\n";
}

uint64_t getVersion() const override {
return 0; // No TZif file.
}

int64_t convertToUTC(int64_t clk) const override {
return clk + variant_.gmtOffset;
}

int64_t convertFromUTC(int64_t clk) const override {
return clk - variant_.gmtOffset;
}

private:
TimezoneVariant variant_;
};

class LazyTimezone : public Timezone {
private:
std::string filename_;
Expand Down Expand Up @@ -766,6 +825,12 @@ namespace orc {
if (itr != timezoneCache.end()) {
return *(itr->second).get();
}
int64_t offset;
if (parseFixedOffset(zone, offset)) {
auto timezone = std::make_shared<FixedOffsetTimezone>(zone, offset);
timezoneCache[filename] = timezone;
return *timezone;
}
auto it = TZ_ALIASES.find(zone);
if (it == TZ_ALIASES.end()) {
timezoneCache[filename] = std::make_shared<LazyTimezone>(filename);
Expand Down
89 changes: 89 additions & 0 deletions c++/test/TestTimezone.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,16 @@
*/

#include "Adaptor.hh"
#include "MemoryInputStream.hh"
#include "Timezone.hh"
#include "orc/OrcFile.hh"
#include "wrap/gmock.h"
#include "wrap/gtest-wrapper.h"

#include <algorithm>
#include <cstdlib>
#include <iostream>
#include <utility>
#include <vector>

namespace orc {
Expand Down Expand Up @@ -363,6 +366,91 @@ namespace orc {
EXPECT_EQ("EST", getVariantFromZone(*ny1, "1974-10-27 06:00:00"));
}

TEST(TestTimezone, testFixedOffsets) {
const std::pair<std::string, int64_t> cases[] = {
{"GMT-00:00", 0}, {"GMT+00:00", 0}, {"GMT+05:30", 19800},
{"GMT-03:30", -12600}, {"GMT+05:45", 20700}, {"GMT+00:10", 600},
{"GMT+23:59", 86340}, {"GMT-23:59", -86340}, {"GMT+05:30:15", 19815},
{"GMT-03:30:15", -12615}, {"GMT+00:00:01", 1}, {"GMT-00:00:59", -59}};
for (const auto& entry : cases) {
SCOPED_TRACE(entry.first);
const auto& zone = getTimezoneByName(entry.first);
EXPECT_EQ(&zone, &getTimezoneByName(entry.first));
EXPECT_EQ(1420070400 - entry.second, zone.getEpoch());
for (int64_t time : {-2208988800LL, 0LL, 1420070400LL, 4102444800LL}) {
const auto& variant = zone.getVariant(time);
EXPECT_EQ(entry.second, variant.gmtOffset);
EXPECT_FALSE(variant.isDst);
EXPECT_EQ(entry.first, variant.name);
EXPECT_EQ(time + entry.second, zone.convertToUTC(time));
EXPECT_EQ(time - entry.second, zone.convertFromUTC(time));
}
}
}

TEST(TestTimezone, testReadJavaFixedOffsets) {
// Written by the Java ORC 2.1.2 writer with OpenJDK 22.0.2. Each file contains
// struct<ts:timestamp>, compression NONE, rowIndexStride 0, and these values:
// 2001-11-12 18:31:01.123456789, 1960-01-02 03:04:05, and
// 2040-07-01 12:00:00.999999999. TimeZone.setDefault is called before creating
// the writer; the seconds offset uses TimeZone.getTimeZone(ZoneOffset.of(...)).
const std::pair<std::string, std::string> files[] = {
{"GMT-00:00",
"T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
"LTAwOjAwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
"OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
{"GMT+05:30",
"T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
"KzA1OjMwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
"OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
{"GMT-03:30",
"T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
"LTAzOjMwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
"OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
{"GMT+05:30:15",
"T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoMR01U"
"KzA1OjMwOjE1ChAKBAgAUAAKCAgASgBQAFgfCAMQSBoKCAMQABgfICYoAyIJCAwSAQEaAnRzIgIICTADOgQI"
"AFAAOggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
};
const int64_t seconds[] = {1005589861, -315521755, 2224756800};
const int64_t nanos[] = {123456789, 0, 999999999};
for (const auto& file : files) {
SCOPED_TRACE(file.first);
auto bytes = decodeBase64(file.second);
auto reader = createReader(std::make_unique<MemoryInputStream>(
reinterpret_cast<const char*>(bytes.data()), bytes.size()),
ReaderOptions());
// Check conversion to both UTC and another fixed offset. In both cases
// the reader must preserve the original wall-clock timestamp.
for (const auto& readerZone : {"GMT+00:00", "GMT-03:30"}) {
SCOPED_TRACE(readerZone);
RowReaderOptions options;
options.setTimezoneName(readerZone);
auto rows = reader->createRowReader(options);
auto batch = rows->createRowBatch(10);
ASSERT_TRUE(rows->next(*batch));
ASSERT_EQ(3, batch->numElements);
const auto& timestamps = dynamic_cast<const TimestampVectorBatch&>(
*dynamic_cast<const StructVectorBatch&>(*batch).fields[0]);
const int64_t readerOffset = std::string(readerZone) == "GMT-03:30" ? -12600 : 0;
for (size_t i = 0; i < 3; ++i) {
EXPECT_EQ(seconds[i] - readerOffset, timestamps.data[i]);
EXPECT_EQ(nanos[i], timestamps.nanoseconds[i]);
}
EXPECT_FALSE(rows->next(*batch));
}
}
}

TEST(TestTimezone, testInvalidFixedOffsets) {
for (const auto& name :
{"GMT+24:00", "GMT-00:60", "GMT+00:00:60", "GMT+05:30x", "GMT+05:30:15x", "GMT+0a:00",
"GMT+00/00", "GMT*05:30", "GMT+5:30", "GMT+05:3", "gmt+05:30", "GMT+"}) {
SCOPED_TRACE(name);
EXPECT_THROW(getTimezoneByName(name).getEpoch(), TimezoneError);
}
}

TEST(TestTimezone, testGMTv1) {
const char GMT[] =
("VFppZgAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAA"
Expand Down Expand Up @@ -437,6 +525,7 @@ namespace orc {
testing::ThrowsMessage<TimezoneError>(testing::HasSubstr(
"Time zone file /path/to/wrong/tzdb/America/Los_Angeles does not exist."
" Please install IANA time zone database and set TZDIR env.")));
EXPECT_EQ(1420050600, getTimezoneByName("GMT+05:30").getEpoch());
if (!tzDirBackup.empty()) {
ASSERT_TRUE(setEnv("TZDIR", tzDirBackup.c_str()));
} else {
Expand Down
Loading