From 25f19f1894587a4d5bda071aa6c9803290ee079b Mon Sep 17 00:00:00 2001 From: Qijia Liu Date: Sun, 27 Sep 2026 00:51:09 -0400 Subject: [PATCH] datrie: use native endian --- src/libime/core/datrie.cpp | 152 ++++++++++--------------------------- src/libime/core/datrie.h | 4 +- src/libime/core/utils_p.h | 26 ------- test/testtrie.cpp | 33 ++++++++ 4 files changed, 73 insertions(+), 142 deletions(-) diff --git a/src/libime/core/datrie.cpp b/src/libime/core/datrie.cpp index 388f8f5..563f542 100644 --- a/src/libime/core/datrie.cpp +++ b/src/libime/core/datrie.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +33,6 @@ #include #include #include -#include "endian_p.h" #include "naivevector.h" #include "utils_p.h" @@ -88,24 +86,6 @@ constexpr T decodeImpl(int32_t raw) { return d.result_value; } -template -void decodeBigEndian(T &data) - requires(sizeof(T) == sizeof(uint32_t)) -{ - auto raw = std::bit_cast(data); - raw = be32toh(raw); - data = std::bit_cast(raw); -} - -template -void decodeBigEndian(T &data) - requires(sizeof(T) == sizeof(uint16_t)) -{ - auto raw = std::bit_cast(data); - raw = be16toh(raw); - data = std::bit_cast(raw); -} - } // namespace // template @@ -142,22 +122,7 @@ class DATriePrivate { node(const int32_t base_ = 0, const int32_t check_ = 0) : base(base_), check(check_) {} - node(std::istream &in) : base(0), check(0) { - throw_if_io_fail(unmarshall(in, base)); - throw_if_io_fail(unmarshall(in, check)); - } - FCITX_INLINE_DEFINE_DEFAULT_DTOR_AND_COPY(node); - - void decode() { - decodeBigEndian(base); - decodeBigEndian(check); - } - - friend std::ostream &operator<<(std::ostream &out, const node &n) { - marshall(out, n.base) && marshall(out, n.check); - return out; - } }; struct npos_t { @@ -193,46 +158,11 @@ class DATriePrivate { int32_t trial; // # trial int32_t ehead; // first empty item block() : prev(0), next(0), num(256), reject(257), trial(0), ehead(0) {} - - block(std::istream &in) : block() { - throw_if_io_fail(unmarshall(in, prev)); - throw_if_io_fail(unmarshall(in, next)); - throw_if_io_fail(unmarshall(in, num)); - throw_if_io_fail(unmarshall(in, reject)); - throw_if_io_fail(unmarshall(in, trial)); - throw_if_io_fail(unmarshall(in, ehead)); - } - - void decode() { - decodeBigEndian(prev); - decodeBigEndian(next); - decodeBigEndian(num); - decodeBigEndian(reject); - decodeBigEndian(trial); - decodeBigEndian(ehead); - } - - friend std::ostream &operator<<(std::ostream &out, const block &b) { - marshall(out, b.prev) && marshall(out, b.next) && - marshall(out, b.num) && marshall(out, b.reject) && - marshall(out, b.trial) && marshall(out, b.ehead); - return out; - } }; struct ninfo { // x1.5 update speed; +.25 % memory (8n -> 10n) uchar sibling; // right sibling (= 0 if not exist) uchar child; // first child ninfo() : sibling(0), child(0) {} - - ninfo(std::istream &in) { - throw_if_io_fail(unmarshall(in, sibling)); - throw_if_io_fail(unmarshall(in, child)); - } - - friend std::ostream &operator<<(std::ostream &out, const ninfo &n) { - marshall(out, n.sibling) && marshall(out, n.child); - return out; - } }; vector_impl m_array; @@ -286,8 +216,9 @@ class DATriePrivate { void open(std::istream &fin) { uint32_t len = 0; uint32_t size_ = 0; - throw_if_io_fail(unmarshall(fin, len)); - throw_if_io_fail(unmarshall(fin, size_)); + throw_if_io_fail(fin.read(reinterpret_cast(&len), sizeof(len))); + throw_if_io_fail( + fin.read(reinterpret_cast(&size_), sizeof(size_))); const size_t length_ = static_cast(len); @@ -303,13 +234,13 @@ class DATriePrivate { throw_if_io_fail(fin.read(reinterpret_cast(m_array.data()), sizeof(node) * size_)); - for (auto &node : m_array) { - node.decode(); - } - throw_if_io_fail(unmarshall(fin, m_bheadF)); - throw_if_io_fail(unmarshall(fin, m_bheadC)); - throw_if_io_fail(unmarshall(fin, m_bheadO)); + throw_if_io_fail( + fin.read(reinterpret_cast(&m_bheadF), sizeof(m_bheadF))); + throw_if_io_fail( + fin.read(reinterpret_cast(&m_bheadC), sizeof(m_bheadC))); + throw_if_io_fail( + fin.read(reinterpret_cast(&m_bheadO), sizeof(m_bheadO))); throw_if_io_fail(fin.read(reinterpret_cast(m_ninfo.data()), sizeof(ninfo) * size_)); @@ -317,9 +248,6 @@ class DATriePrivate { throw_if_io_fail(fin.read(reinterpret_cast(m_block.data()), sizeof(block) * blockSize)); - for (auto &block : m_block) { - block.decode(); - } } void save(std::ostream &fout) { @@ -329,30 +257,28 @@ class DATriePrivate { const uint32_t size_ = size(); assert(m_block.size() << 8 == m_ninfo.size()); - throw_if_io_fail(marshall(fout, length)); - throw_if_io_fail(marshall(fout, size_)); - throw_if_io_fail(fout.write(reinterpret_cast(m_tail.data()), - sizeof(char) * length)); - - auto s = size_; - for (auto &n : m_array) { - throw_if_io_fail(fout << n); - if (--s == 0) { - break; - } - } - - throw_if_io_fail(marshall(fout, m_bheadF)); - throw_if_io_fail(marshall(fout, m_bheadC)); - throw_if_io_fail(marshall(fout, m_bheadO)); - - for (auto &n : m_ninfo) { - throw_if_io_fail(fout << n); - } - - for (auto &b : m_block) { - throw_if_io_fail(fout << b); - } + throw_if_io_fail(fout.write(reinterpret_cast(&length), + sizeof(length))); + throw_if_io_fail( + fout.write(reinterpret_cast(&size_), sizeof(size_))); + throw_if_io_fail( + fout.write(reinterpret_cast(m_tail.data()), + sizeof(char) * length)); + throw_if_io_fail( + fout.write(reinterpret_cast(m_array.data()), + sizeof(node) * size_)); + throw_if_io_fail(fout.write(reinterpret_cast(&m_bheadF), + sizeof(m_bheadF))); + throw_if_io_fail(fout.write(reinterpret_cast(&m_bheadC), + sizeof(m_bheadC))); + throw_if_io_fail(fout.write(reinterpret_cast(&m_bheadO), + sizeof(m_bheadO))); + throw_if_io_fail( + fout.write(reinterpret_cast(m_ninfo.data()), + sizeof(ninfo) * size_)); + throw_if_io_fail( + fout.write(reinterpret_cast(m_block.data()), + sizeof(block) * m_block.size())); } void init() { @@ -459,7 +385,7 @@ class DATriePrivate { npos.offset = offset + moved; } char *const data = &tail[len + 1]; - storeDWord(data, callback(loadDWord(data))); + storeNative(data, callback(loadNative(data))); return; } // otherwise, insert the common prefix in tail if any @@ -509,7 +435,7 @@ class DATriePrivate { // tail[pos] == 0, so the actual content in tail can be get rid // of // and tail0 is used to track those hole - m_array[to].value = loadDWord(&tail[pos + 1]); + m_array[to].value = loadNative(&tail[pos + 1]); } from = static_cast( _follow(from, static_cast(key[pos]), cf)); @@ -522,7 +448,7 @@ class DATriePrivate { m_array[from].base = -offset0; m_tail0.pop_back(); char *const data = &m_tail[offset0 + 1]; - storeDWord(data, callback(0)); + storeNative(data, callback(0)); return; } if (m_tail.capacity() < m_tail.size() + needed) { @@ -543,7 +469,7 @@ class DATriePrivate { npos.offset = old_length + len - pos_orig; } char *const data = &tail[len + 1]; - storeDWord(data, callback(loadDWord(data))); + storeNative(data, callback(loadNative(data))); } // easy-going erase () without compression @@ -619,8 +545,8 @@ class DATriePrivate { t.push_back(tail_[i]); } while (tail_[i++]); t.resize(t.size() + sizeof(value_type)); - storeDWord(&t[t.size() - sizeof(value_type)], - loadDWord(&tail_[i])); + storeNative(&t[t.size() - sizeof(value_type)], + loadNative(&tail_[i])); } } using std::swap; @@ -654,7 +580,7 @@ class DATriePrivate { const size_t len_ = std::strlen(&m_tail[-base]); npos.offset = static_cast(-base) + len_; len += len_; - return loadDWord(&m_tail[-base] + len_ + 1); + return loadNative(&m_tail[-base] + len_ + 1); } // return the next child if any int32_t next(npos_t &npos, size_t &len, @@ -740,7 +666,7 @@ class DATriePrivate { if (tail[pos]) { return CEDAR_NO_VALUE; // input < tail } - return loadDWord(&tail[len + 1]); + return loadNative(&tail[len + 1]); } // explore new block to settle down diff --git a/src/libime/core/datrie.h b/src/libime/core/datrie.h index fab91a2..1bf3715 100644 --- a/src/libime/core/datrie.h +++ b/src/libime/core/datrie.h @@ -44,9 +44,7 @@ struct NaN { /** * This is a trie based on cedar. * - * comparing with original version, this version takes care of endianness when - * doing serialization, and handle the possible unaligned memory access on some - * platform. + * Serialized tries use the host byte order and ABI. * * It also exploits the C++11 feature which can let you implement update easily * with lambda, comparing with the original version's add/set. diff --git a/src/libime/core/utils_p.h b/src/libime/core/utils_p.h index 0356807..4ccbe34 100644 --- a/src/libime/core/utils_p.h +++ b/src/libime/core/utils_p.h @@ -30,32 +30,6 @@ inline void storeNative(char *data, T v) { memcpy(data, &v, sizeof(v)); } -// These are only used by DATrie, and it's le32 due to historic reason. -// Marshall* still use be. -template -inline T loadDWord(const char *data) { - static_assert(sizeof(T) == sizeof(uint32_t), ""); - union { - uint32_t i; - T v; - } c; - memcpy(&c, data, sizeof(c)); - c.i = le32toh(c.i); - return c.v; -} - -template -inline void storeDWord(char *data, T v) { - static_assert(sizeof(T) == sizeof(uint32_t), ""); - union { - uint32_t i; - T v; - } c; - c.v = v; - c.i = htole32(c.i); - memcpy(data, &c, sizeof(c)); -} - static inline std::istream &unmarshallLE(std::istream &in, uint32_t &data) { in.read(reinterpret_cast(&data), sizeof(data)); data = le32toh(data); diff --git a/test/testtrie.cpp b/test/testtrie.cpp index 72eee36..a898141 100644 --- a/test/testtrie.cpp +++ b/test/testtrie.cpp @@ -5,6 +5,7 @@ */ #include #include +#include #include #include "libime/core/datrie.h" @@ -70,5 +71,37 @@ int main() { trie.erase(pos); FCITX_ASSERT(trie.size() == 4); } + + { + DATrie source; + source.set("hello|world", 1.25F); + source.set("hello|there", 2.5F); + source.set("help|me", 3.75F); + + std::stringstream stream(std::ios::in | std::ios::out | + std::ios::binary); + source.save(stream); + + const auto serialized = stream.str(); + uint32_t tailSize; + uint32_t nodeCount; + std::memcpy(&tailSize, serialized.data(), sizeof(tailSize)); + std::memcpy(&nodeCount, serialized.data() + sizeof(tailSize), + sizeof(nodeCount)); + FCITX_ASSERT(nodeCount % 256 == 0); + FCITX_ASSERT(serialized.size() == + sizeof(tailSize) + sizeof(nodeCount) + tailSize + + nodeCount * 8 + 3 * sizeof(int32_t) + nodeCount * 2 + + (nodeCount / 256) * 20); + + stream.seekg(0); + + DATrie loaded; + loaded.load(stream); + FCITX_ASSERT(loaded.size() == source.size()); + FCITX_ASSERT(loaded.exactMatchSearch("hello|world") == 1.25F); + FCITX_ASSERT(loaded.exactMatchSearch("hello|there") == 2.5F); + FCITX_ASSERT(loaded.exactMatchSearch("help|me") == 3.75F); + } return 0; }