From 6e23a8700ad9d5be5b55750c09f26398a769ef8c Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 12:06:03 -0400 Subject: [PATCH 1/6] feat(pagination): add generic cursor and offset paginator --- cmake/PolymarketTests.cmake | 4 + include/polymarket/pagination.hpp | 298 ++++++++++++++++++++++++++++++ src/clob_account.cpp | 2 +- src/clob_activity_pagination.cpp | 2 +- src/opaque_cursor_pagination.hpp | 59 ------ tests/test_pagination.cpp | 257 ++++++++++++++++++++++++++ 6 files changed, 561 insertions(+), 61 deletions(-) create mode 100644 include/polymarket/pagination.hpp delete mode 100644 src/opaque_cursor_pagination.hpp create mode 100644 tests/test_pagination.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index 000502f..f4f0de1 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -175,6 +175,10 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_prepared_order PRIVATE polymarket::client) add_test(NAME test_prepared_order COMMAND test_prepared_order) + add_executable(test_pagination tests/test_pagination.cpp) + target_link_libraries(test_pagination PRIVATE polymarket::client) + add_test(NAME test_pagination COMMAND test_pagination) + add_executable(test_sdk_error tests/test_sdk_error.cpp) target_link_libraries(test_sdk_error PRIVATE polymarket::client) add_test(NAME test_sdk_error COMMAND test_sdk_error) diff --git a/include/polymarket/pagination.hpp b/include/polymarket/pagination.hpp new file mode 100644 index 0000000..0764910 --- /dev/null +++ b/include/polymarket/pagination.hpp @@ -0,0 +1,298 @@ +#pragma once + +#include "polymarket/sdk_error.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace polymarket +{ + namespace detail + { + class OpaqueCursorPaginationError : public std::runtime_error + { + public: + using std::runtime_error::runtime_error; + }; + + class OpaqueCursorPagination + { + public: + static constexpr size_t max_pages = 1000; + + OpaqueCursorPagination(std::string initial_cursor, std::string end_cursor) + : cursor_(std::move(initial_cursor)), end_cursor_(std::move(end_cursor)) + { + } + + const std::string &cursor() const { return cursor_; } + + bool begin_page() + { + if (cursor_.empty() || cursor_ == end_cursor_) return false; + if (pages_started_ >= max_pages) + throw OpaqueCursorPaginationError( + "opaque cursor pagination exceeded 1000 pages"); + if (!seen_cursors_.insert(cursor_).second) + throw OpaqueCursorPaginationError("opaque cursor pagination cycle detected"); + ++pages_started_; + return true; + } + + void advance(std::string next_cursor) + { + if (next_cursor.empty() || next_cursor == end_cursor_) + { + cursor_ = end_cursor_; + return; + } + if (seen_cursors_.contains(next_cursor)) + throw OpaqueCursorPaginationError("opaque cursor pagination cycle detected"); + cursor_ = std::move(next_cursor); + } + + private: + std::string cursor_; + std::string end_cursor_; + size_t pages_started_{0}; + std::unordered_set seen_cursors_; + }; + } // namespace detail + + // The next page to request. Cursor pagination fills `cursor`; offset + // pagination fills `offset` and `limit`. Pass it back to resume later. + struct PageRequest + { + std::string cursor; + size_t offset{0}; + size_t limit{0}; + }; + + // One fetched page. `next_cursor` is read only by cursor pagination. + template struct Page + { + std::vector items; + std::string next_cursor; + }; + + // Pulls pages lazily through `fetch`. Cursor pagination ends at an empty or + // end cursor and fails on a cycle or after max_pages. Offset pagination ends + // on a short page and stops with truncated() set once the next offset would + // pass `max_offset`; items inserted between requests can shift across pages. + // After a failure every call reports the same error. Not thread-safe; any + // client captured by `fetch` must outlive the paginator. + template class Paginator + { + public: + using Fetch = std::function>(const PageRequest &)>; + class ItemRange; + + static Paginator cursor(std::string endpoint, Fetch fetch, std::string initial_cursor, + std::string end_cursor) + { + Paginator paginator(std::move(endpoint), std::move(fetch)); + paginator.cursor_.emplace(std::move(initial_cursor), std::move(end_cursor)); + return paginator; + } + + static Paginator offset(std::string endpoint, Fetch fetch, size_t limit, size_t max_offset, + size_t start_offset = 0) + { + if (limit == 0) throw std::invalid_argument("pagination limit must be positive"); + Paginator paginator(std::move(endpoint), std::move(fetch)); + paginator.limit_ = limit; + paginator.max_offset_ = max_offset; + paginator.offset_ = start_offset; + paginator.truncated_ = start_offset > max_offset; + paginator.done_ = paginator.truncated_; + return paginator; + } + + // nullopt once exhausted. A page may be empty. + Result>> next_page() + { + using PageResult = Result>>; + if (error_) return PageResult::failure(*error_); + if (done_) return PageResult::success(std::nullopt); + return cursor_ ? next_cursor_page() : next_offset_page(); + } + + // Drains remaining pages, keeping at most `max_items`. resume_token() + // then points past the last fetched page, not past the last kept item. + Result> collect(size_t max_items = std::numeric_limits::max()) + { + std::vector items; + while (items.size() < max_items) + { + auto page = next_page(); + if (!page.ok()) return Result>::failure(page.error()); + if (!page.value()) break; + auto &batch = *page.value(); + const size_t keep = std::min(batch.size(), max_items - items.size()); + items.insert( + items.end(), std::make_move_iterator(batch.begin()), + std::make_move_iterator(batch.begin() + static_cast(keep))); + } + return Result>::success(std::move(items)); + } + + // Range-for over remaining items. Iteration stops on failure; check + // error() afterwards. After a break, items() resumes at the item last + // returned. next_page() and collect() skip items still buffered here. + // The paginator must stay in place while iterating. + ItemRange items() { return ItemRange(*this); } + + const std::optional &error() const { return error_; } + bool truncated() const { return truncated_; } + bool done() const { return done_ || error_.has_value(); } + + PageRequest resume_token() const + { + if (cursor_) return PageRequest{cursor_->cursor(), 0, 0}; + return PageRequest{"", offset_, limit_}; + } + + class ItemIterator + { + public: + using value_type = T; + using difference_type = std::ptrdiff_t; + + ItemIterator() = default; + explicit ItemIterator(Paginator &paginator) : paginator_(&paginator) + { + if (paginator_->buffer_index_ >= paginator_->buffer_.size()) + paginator_->fill_buffer(); + } + + T &operator*() const { return paginator_->buffer_[paginator_->buffer_index_]; } + T *operator->() const { return &**this; } + + ItemIterator &operator++() + { + if (++paginator_->buffer_index_ >= paginator_->buffer_.size()) + paginator_->fill_buffer(); + return *this; + } + void operator++(int) { ++*this; } + + friend bool operator==(const ItemIterator &it, std::default_sentinel_t) + { + return it.paginator_ == nullptr || + it.paginator_->buffer_index_ >= it.paginator_->buffer_.size(); + } + + private: + Paginator *paginator_{nullptr}; + }; + + class ItemRange + { + public: + explicit ItemRange(Paginator &paginator) : paginator_(&paginator) {} + ItemIterator begin() const { return ItemIterator(*paginator_); } + std::default_sentinel_t end() const { return {}; } + + private: + Paginator *paginator_; + }; + + private: + Paginator(std::string endpoint, Fetch fetch) + : endpoint_(std::move(endpoint)), fetch_(std::move(fetch)) + { + if (!fetch_) throw std::invalid_argument("pagination fetch must be set"); + } + + Result>> fail(SdkError error) + { + error_ = std::move(error); + return Result>>::failure(*error_); + } + + Result>> next_cursor_page() + { + using PageResult = Result>>; + try + { + if (!cursor_->begin_page()) + { + done_ = true; + return PageResult::success(std::nullopt); + } + } + catch (const detail::OpaqueCursorPaginationError &ex) + { + return fail(make_parse_error(ex.what(), endpoint_)); + } + + auto page = fetch_(PageRequest{cursor_->cursor(), 0, 0}); + if (!page.ok()) return fail(page.error()); + try + { + cursor_->advance(std::move(page.value().next_cursor)); + } + catch (const detail::OpaqueCursorPaginationError &ex) + { + return fail(make_parse_error(ex.what(), endpoint_)); + } + return PageResult::success(std::move(page.value().items)); + } + + Result>> next_offset_page() + { + using PageResult = Result>>; + auto page = fetch_(PageRequest{"", offset_, limit_}); + if (!page.ok()) return fail(page.error()); + auto &items = page.value().items; + if (items.size() > limit_) + return fail(make_parse_error("page holds " + std::to_string(items.size()) + + " items, more than limit " + + std::to_string(limit_), + endpoint_)); + if (items.size() < limit_) + done_ = true; + else if (max_offset_ - offset_ < limit_) + done_ = truncated_ = true; + else + offset_ += limit_; + return PageResult::success(std::move(items)); + } + + void fill_buffer() + { + buffer_.clear(); + buffer_index_ = 0; + while (true) + { + auto page = next_page(); + if (!page.ok() || !page.value()) return; + if (!page.value()->empty()) + { + buffer_ = std::move(*page.value()); + return; + } + } + } + + std::string endpoint_; + Fetch fetch_; + std::optional cursor_; + size_t offset_{0}; + size_t limit_{0}; + size_t max_offset_{0}; + bool done_{false}; + bool truncated_{false}; + std::optional error_; + std::vector buffer_; + size_t buffer_index_{0}; + }; +} // namespace polymarket diff --git a/src/clob_account.cpp b/src/clob_account.cpp index ebb63bf..a991973 100644 --- a/src/clob_account.cpp +++ b/src/clob_account.cpp @@ -1,6 +1,6 @@ #include "polymarket/clob_client.hpp" #include "clob_client_internal.hpp" -#include "opaque_cursor_pagination.hpp" +#include "polymarket/pagination.hpp" #include #include diff --git a/src/clob_activity_pagination.cpp b/src/clob_activity_pagination.cpp index b2d85d3..6808873 100644 --- a/src/clob_activity_pagination.cpp +++ b/src/clob_activity_pagination.cpp @@ -1,6 +1,6 @@ #include "polymarket/clob_client.hpp" #include "clob_client_internal.hpp" -#include "opaque_cursor_pagination.hpp" +#include "polymarket/pagination.hpp" #include diff --git a/src/opaque_cursor_pagination.hpp b/src/opaque_cursor_pagination.hpp deleted file mode 100644 index 22262c5..0000000 --- a/src/opaque_cursor_pagination.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace polymarket::detail -{ - class OpaqueCursorPaginationError : public std::runtime_error - { - public: - using std::runtime_error::runtime_error; - }; - - class OpaqueCursorPagination - { - public: - static constexpr size_t MAX_PAGES = 1000; - - OpaqueCursorPagination(std::string initial_cursor, std::string end_cursor) - : cursor_(std::move(initial_cursor)), end_cursor_(std::move(end_cursor)) - { - } - - const std::string &cursor() const { return cursor_; } - - bool begin_page() - { - if (cursor_.empty() || cursor_ == end_cursor_) - return false; - if (pages_started_ >= MAX_PAGES) - throw OpaqueCursorPaginationError("opaque cursor pagination exceeded 1000 pages"); - if (!seen_cursors_.insert(cursor_).second) - throw OpaqueCursorPaginationError("opaque cursor pagination cycle detected"); - ++pages_started_; - return true; - } - - void advance(std::string next_cursor) - { - if (next_cursor.empty() || next_cursor == end_cursor_) - { - cursor_ = end_cursor_; - return; - } - if (seen_cursors_.contains(next_cursor)) - throw OpaqueCursorPaginationError("opaque cursor pagination cycle detected"); - cursor_ = std::move(next_cursor); - } - - private: - std::string cursor_; - std::string end_cursor_; - size_t pages_started_{0}; - std::unordered_set seen_cursors_; - }; -} diff --git a/tests/test_pagination.cpp b/tests/test_pagination.cpp new file mode 100644 index 0000000..cdada3b --- /dev/null +++ b/tests/test_pagination.cpp @@ -0,0 +1,257 @@ +#include "check_support.hpp" +#include "polymarket/pagination.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using check_support::check; + using check_support::expect_equal; + using polymarket::Page; + using polymarket::PageRequest; + using polymarket::Paginator; + using polymarket::Result; + using polymarket::SdkError; + using polymarket::SdkErrorCode; + + using IntPaginator = Paginator; + static_assert(std::input_iterator); + static_assert(std::ranges::input_range); + + struct ScriptedFetch + { + std::vector>> pages; + std::vector requests; + size_t next{0}; + + IntPaginator::Fetch fn() + { + return [this](const PageRequest &request) + { + requests.push_back(request); + if (next >= pages.size()) throw std::logic_error("unexpected page request"); + return pages[next++]; + }; + } + }; + + Result> page(std::vector items, std::string next_cursor = "") + { + return Result>::success(Page{std::move(items), std::move(next_cursor)}); + } + + Result> http_failure() + { + SdkError error; + error.code = SdkErrorCode::HttpTransport; + error.message = "connection reset"; + error.endpoint = "/items"; + return Result>::failure(error); + } + + std::vector drain(IntPaginator &paginator) + { + std::vector items; + for (int item : paginator.items()) + items.push_back(item); + return items; + } + + void test_cursor_follows_until_end_cursor() + { + ScriptedFetch fetch{{page({1, 2}, "A"), page({3}, "LTE=")}}; + auto paginator = IntPaginator::cursor("/items", fetch.fn(), "MA==", "LTE="); + check(drain(paginator) == std::vector({1, 2, 3}), + "cursor items must cover every page"); + check(fetch.requests.size() == 2, "cursor must stop at the end cursor"); + expect_equal("first cursor", fetch.requests[0].cursor, "MA=="); + expect_equal("second cursor", fetch.requests[1].cursor, "A"); + check(paginator.done() && !paginator.error() && !paginator.truncated(), + "cursor must end cleanly"); + auto after = paginator.next_page(); + check(after.ok() && !after.value(), "exhausted cursor must return nullopt"); + } + + void test_cursor_empty_next_cursor_ends() + { + ScriptedFetch fetch{{page({1}, "")}}; + auto paginator = IntPaginator::cursor("/items", fetch.fn(), "MA==", "LTE="); + auto collected = paginator.collect(); + check(collected.ok() && collected.value() == std::vector({1}), + "empty cursor must end paging"); + check(fetch.requests.size() == 1, "empty cursor must not request again"); + } + + void test_cursor_cycle_fails() + { + ScriptedFetch fetch{{page({1}, "A"), page({2}, "A")}}; + auto paginator = IntPaginator::cursor("/items", fetch.fn(), "MA==", "LTE="); + auto first = paginator.next_page(); + check(first.ok() && first.value() && *first.value() == std::vector({1}), + "first page must load"); + auto second = paginator.next_page(); + check(!second.ok() && second.error().code == SdkErrorCode::Parse, + "cursor cycle must fail as parse"); + expect_equal("cycle endpoint", second.error().endpoint, "/items"); + check(paginator.error().has_value() && paginator.done(), "cycle must set error()"); + } + + void test_cursor_page_cap_fails() + { + size_t requests = 0; + auto paginator = IntPaginator::cursor( + "/items", + [&](const PageRequest &) + { + ++requests; + return page({1}, "C" + std::to_string(requests)); + }, + "MA==", "LTE="); + auto collected = paginator.collect(); + check(!collected.ok() && collected.error().code == SdkErrorCode::Parse, + "page cap must fail"); + check(requests == polymarket::detail::OpaqueCursorPagination::max_pages, + "page cap must stop requests at max_pages"); + } + + void test_offset_stops_on_short_page() + { + ScriptedFetch fetch{{page({1, 2}), page({3, 4}), page({5})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 10'000); + check(drain(paginator) == std::vector({1, 2, 3, 4, 5}), + "offset items must cover every page"); + check(fetch.requests.size() == 3, "short page must end offset paging"); + check(fetch.requests[0].offset == 0 && fetch.requests[1].offset == 2 && + fetch.requests[2].offset == 4, + "offset must advance by limit"); + check(fetch.requests[0].limit == 2, "offset request must carry the limit"); + check(!paginator.truncated() && !paginator.error(), "short page is a clean end"); + } + + void test_offset_cap_sets_truncated() + { + ScriptedFetch fetch{{page({1, 2}), page({3, 4})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 2); + auto collected = paginator.collect(); + check(collected.ok() && collected.value() == std::vector({1, 2, 3, 4}), + "pages up to the cap load"); + check(fetch.requests.size() == 2 && fetch.requests[1].offset == 2, + "max_offset itself is requested"); + check(paginator.truncated() && paginator.done() && !paginator.error(), + "offset cap must set truncated()"); + } + + void test_offset_oversized_page_fails() + { + ScriptedFetch fetch{{page({1, 2, 3})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 10'000); + auto first = paginator.next_page(); + check(!first.ok() && first.error().code == SdkErrorCode::Parse, + "page above limit must fail"); + } + + void test_offset_rejects_zero_limit() + { + ScriptedFetch fetch; + check_support::expect_throws( + "zero limit", [&] { (void)IntPaginator::offset("/items", fetch.fn(), 0, 10); }); + check_support::expect_throws( + "missing fetch", [] { (void)IntPaginator::cursor("/items", {}, "MA==", "LTE="); }); + } + + void test_failure_keeps_earlier_pages_and_sticks() + { + ScriptedFetch fetch{{page({1, 2}, "A"), http_failure()}}; + auto paginator = IntPaginator::cursor("/items", fetch.fn(), "MA==", "LTE="); + check(drain(paginator) == std::vector({1, 2}), + "items before a failure must be yielded"); + check(paginator.error() && paginator.error()->code == SdkErrorCode::HttpTransport, + "range-for must expose the fetch error"); + auto again = paginator.next_page(); + check(!again.ok() && again.error().message == "connection reset", "failure must be sticky"); + check(fetch.requests.size() == 2, "failed paginator must not fetch again"); + } + + void test_collect_reports_failure() + { + ScriptedFetch fetch{{page({1}), http_failure()}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 1, 10); + auto collected = paginator.collect(); + check(!collected.ok() && collected.error().code == SdkErrorCode::HttpTransport, + "collect must fail when a later page fails"); + } + + void test_collect_max_items() + { + ScriptedFetch fetch{{page({1, 2}), page({3, 4}), page({5})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 10'000); + auto collected = paginator.collect(3); + check(collected.ok() && collected.value() == std::vector({1, 2, 3}), + "collect must stop at max_items"); + check(fetch.requests.size() == 2, "collect must not fetch past max_items"); + check(paginator.resume_token().offset == 4, + "resume token must point past the last fetched page"); + } + + void test_resume_tokens() + { + ScriptedFetch cursor_fetch{{page({1}, "B"), page({2}, "LTE=")}}; + auto cursor = IntPaginator::cursor("/items", cursor_fetch.fn(), "MA==", "LTE="); + (void)cursor.next_page(); + const auto saved = cursor.resume_token(); + expect_equal("saved cursor", saved.cursor, "B"); + auto resumed = IntPaginator::cursor("/items", cursor_fetch.fn(), saved.cursor, "LTE="); + check(drain(resumed) == std::vector({2}), "resumed cursor must continue"); + + ScriptedFetch offset_fetch{{page({7})}}; + auto offset = IntPaginator::offset("/items", offset_fetch.fn(), 2, 10, 4); + check(drain(offset) == std::vector({7}), "offset must start from start_offset"); + check(offset_fetch.requests[0].offset == 4, "start_offset must be requested first"); + + ScriptedFetch past_cap; + auto beyond = IntPaginator::offset("/items", past_cap.fn(), 2, 10, 12); + check(beyond.truncated() && drain(beyond).empty() && past_cap.requests.empty(), + "start past max_offset must not fetch"); + } + + void test_range_skips_empty_pages_and_resumes_after_break() + { + ScriptedFetch fetch{ + {page({}, "A"), page({1, 2, 3}, "B"), page({}, "C"), page({4}, "LTE=")}}; + auto paginator = IntPaginator::cursor("/items", fetch.fn(), "MA==", "LTE="); + std::vector seen; + for (int item : paginator.items()) + { + seen.push_back(item); + if (item == 2) break; + } + for (int item : paginator.items()) + seen.push_back(item); + check(seen == std::vector({1, 2, 2, 3, 4}), + "range must skip empty pages and resume at the break"); + check(paginator.done() && !paginator.error(), "range must end cleanly"); + } +} // namespace + +int main() +{ + test_cursor_follows_until_end_cursor(); + test_cursor_empty_next_cursor_ends(); + test_cursor_cycle_fails(); + test_cursor_page_cap_fails(); + test_offset_stops_on_short_page(); + test_offset_cap_sets_truncated(); + test_offset_oversized_page_fails(); + test_offset_rejects_zero_limit(); + test_failure_keeps_earlier_pages_and_sticks(); + test_collect_reports_failure(); + test_collect_max_items(); + test_resume_tokens(); + test_range_skips_empty_pages_and_resumes_after_break(); + return check_support::finish("test_pagination"); +} From e87e26c8354c94fc5deb02fc254b74a3d96fe206 Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 12:45:35 -0400 Subject: [PATCH 2/6] refactor(clob): route cursor and offset loops through Paginator --- src/clob_account.cpp | 119 ++++++++----------- src/clob_activity_pagination.cpp | 192 ++++++++++++------------------- src/clob_position_pages.cpp | 43 ++++--- 3 files changed, 147 insertions(+), 207 deletions(-) diff --git a/src/clob_account.cpp b/src/clob_account.cpp index a991973..74031e3 100644 --- a/src/clob_account.cpp +++ b/src/clob_account.cpp @@ -141,86 +141,69 @@ namespace polymarket return Result>::failure(make_auth_error("Client not authenticated", endpoint)); } - std::vector orders; - OpaqueCursorPagination pagination(INITIAL_CURSOR, END_CURSOR); - while (true) - { - try - { - if (!pagination.begin_page()) - break; - } - catch (const OpaqueCursorPaginationError &ex) + auto paginator = Paginator::cursor( + endpoint, + [&](const PageRequest &request) -> Result> { - return Result>::failure(make_parse_error(ex.what(), endpoint)); - } - - std::string path = endpoint; - path += market.empty() ? "?" : "?market=" + percent_encode_query_value(market) + "&"; - path += "next_cursor=" + percent_encode_query_value(pagination.cursor()); + std::string path = endpoint; + path += + market.empty() ? "?" : "?market=" + percent_encode_query_value(market) + "&"; + path += "next_cursor=" + percent_encode_query_value(request.cursor); - auto response = - read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); + auto response = + read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); - if (!response.ok()) - { - return Result>::failure(make_sdk_error(response, endpoint)); - } - - try - { - auto page = detail::parse_open_order_page_json(response.body); - orders.insert(orders.end(), page.orders.begin(), page.orders.end()); - pagination.advance(std::move(page.next_cursor)); - } - catch (const std::exception &ex) - { - return Result>::failure(make_parse_error(ex.what(), endpoint, response.body)); - } - } + if (!response.ok()) + { + return Result>::failure(make_sdk_error(response, endpoint)); + } - return Result>::success(std::move(orders)); + try + { + auto page = detail::parse_open_order_page_json(response.body); + return Result>::success( + Page{std::move(page.orders), std::move(page.next_cursor)}); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + INITIAL_CURSOR, END_CURSOR); + return paginator.collect(); } std::vector ClobClient::get_trades(const std::string &next_cursor) { constexpr const char *endpoint = "/data/trades"; - std::vector trades; - OpaqueCursorPagination pagination(next_cursor.empty() ? INITIAL_CURSOR : next_cursor, - END_CURSOR); - while (true) - { - try - { - if (!pagination.begin_page()) - break; - } - catch (const OpaqueCursorPaginationError &) + auto paginator = Paginator::cursor( + endpoint, + [&](const PageRequest &request) -> Result> { - return {}; - } - - const std::string path = std::string(endpoint) + - "?next_cursor=" + percent_encode_query_value(pagination.cursor()); - auto response = - read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); + const std::string path = std::string(endpoint) + "?next_cursor=" + + percent_encode_query_value(request.cursor); + auto response = + read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); - if (!response.ok()) - return {}; + if (!response.ok()) + return Result>::failure(make_sdk_error(response, endpoint)); - try - { - auto page = detail::parse_trade_page_json(response.body); - trades.insert(trades.end(), page.trades.begin(), page.trades.end()); - pagination.advance(std::move(page.next_cursor)); - } - catch (...) - { - return {}; - } - } - - return trades; + try + { + auto page = detail::parse_trade_page_json(response.body); + return Result>::success( + Page{std::move(page.trades), std::move(page.next_cursor)}); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + next_cursor.empty() ? INITIAL_CURSOR : next_cursor, END_CURSOR); + auto trades = paginator.collect(); + return trades ? std::move(trades.value()) : std::vector{}; } std::optional ClobClient::get_balance_allowance( diff --git a/src/clob_activity_pagination.cpp b/src/clob_activity_pagination.cpp index 6808873..ceb317e 100644 --- a/src/clob_activity_pagination.cpp +++ b/src/clob_activity_pagination.cpp @@ -4,7 +4,8 @@ #include -#include +#include +#include using json = nlohmann::json; @@ -12,50 +13,60 @@ namespace polymarket { using namespace detail; - std::vector ClobClient::get_rewards_markets_current() + namespace { - std::vector result; - OpaqueCursorPagination pagination(INITIAL_CURSOR, END_CURSOR); - while (true) + template + std::vector collect_reward_pages(const std::string &endpoint, Get get, + ParseItem parse_item) { - try - { - if (!pagination.begin_page()) - break; - } - catch (const OpaqueCursorPaginationError &) - { - return {}; - } - - auto response = read( - [&] + auto paginator = Paginator::cursor( + endpoint, + [&](const PageRequest &request) -> Result> { - return http_.get("/rewards/markets/current?next_cursor=" + - percent_encode_query_value(pagination.cursor())); - }); - if (!response.ok()) - return {}; - try - { - const auto page = json::parse(response.body); - if (!page.is_object() || !page.at("data").is_array() || - !page.at("next_cursor").is_string()) - return {}; - std::vector parsed_page; - parsed_page.reserve(page.at("data").size()); - for (const auto &item : page.at("data")) - parsed_page.push_back(parse_current_reward_info(item)); - result.insert(result.end(), std::make_move_iterator(parsed_page.begin()), - std::make_move_iterator(parsed_page.end())); - pagination.advance(page.at("next_cursor").get()); - } - catch (...) - { - return {}; - } + const auto response = get(request.cursor); + if (!response.ok()) + return Result>::failure(make_sdk_error(response, endpoint)); + try + { + const json body = json::parse(response.body); + if (!body.is_object() || !body.at("data").is_array() || + !body.at("next_cursor").is_string()) + return Result>::failure( + make_parse_error("rewards page must hold data and next_cursor", + endpoint, response.body)); + Page page; + page.items.reserve(body.at("data").size()); + for (const auto &item : body.at("data")) + page.items.push_back(parse_item(item)); + page.next_cursor = body.at("next_cursor").get(); + return Result>::success(std::move(page)); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + INITIAL_CURSOR, END_CURSOR); + auto items = paginator.collect(); + return items ? std::move(items.value()) : std::vector{}; } - return result; + } // namespace + + std::vector ClobClient::get_rewards_markets_current() + { + return collect_reward_pages( + "/rewards/markets/current", + [this](const std::string &cursor) + { + return read( + [&] + { + return http_.get("/rewards/markets/current?next_cursor=" + + percent_encode_query_value(cursor)); + }); + }, + parse_current_reward_info); } std::vector ClobClient::get_rewards_markets( @@ -64,46 +75,16 @@ namespace polymarket if (condition_id.empty()) return {}; - std::vector result; - OpaqueCursorPagination pagination(INITIAL_CURSOR, END_CURSOR); - while (true) - { - try - { - if (!pagination.begin_page()) - break; - } - catch (const OpaqueCursorPaginationError &) - { - return {}; - } - - const std::string path = "/rewards/markets/" + condition_id + - "?next_cursor=" + - percent_encode_query_value(pagination.cursor()); - auto response = read([&] { return http_.get(path); }); - if (!response.ok()) - return {}; - try + const std::string endpoint = "/rewards/markets/" + condition_id; + return collect_reward_pages( + endpoint, + [this, &endpoint](const std::string &cursor) { - const auto page = json::parse(response.body); - if (!page.is_object() || !page.at("data").is_array() || - !page.at("next_cursor").is_string()) - return {}; - std::vector parsed_page; - parsed_page.reserve(page.at("data").size()); - for (const auto &item : page.at("data")) - parsed_page.push_back(parse_market_reward_info(item)); - result.insert(result.end(), std::make_move_iterator(parsed_page.begin()), - std::make_move_iterator(parsed_page.end())); - pagination.advance(page.at("next_cursor").get()); - } - catch (...) - { - return {}; - } - } - return result; + const std::string path = + endpoint + "?next_cursor=" + percent_encode_query_value(cursor); + return read([&] { return http_.get(path); }); + }, + parse_market_reward_info); } std::optional ClobClient::get_earnings_for_user_for_day( @@ -119,47 +100,16 @@ namespace polymarket const std::string &date) { constexpr const char *endpoint = "/rewards/user"; - std::vector result; - OpaqueCursorPagination pagination(INITIAL_CURSOR, END_CURSOR); - while (true) - { - try + return collect_reward_pages( + endpoint, + [this, &date](const std::string &cursor) { - if (!pagination.begin_page()) - break; - } - catch (const OpaqueCursorPaginationError &) - { - return {}; - } - - const std::string path = std::string(endpoint) + - "?date=" + percent_encode_query_value(date) + - "&signature_type=" + std::to_string(static_cast(sig_type_)) + - "&next_cursor=" + percent_encode_query_value(pagination.cursor()); - auto response = - read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); - if (!response.ok()) - return {}; - try - { - const auto page = json::parse(response.body); - if (!page.is_object() || !page.at("data").is_array() || - !page.at("next_cursor").is_string()) - return {}; - std::vector parsed_page; - parsed_page.reserve(page.at("data").size()); - for (const auto &item : page.at("data")) - parsed_page.push_back(parse_user_earning_info(item)); - result.insert(result.end(), std::make_move_iterator(parsed_page.begin()), - std::make_move_iterator(parsed_page.end())); - pagination.advance(page.at("next_cursor").get()); - } - catch (...) - { - return {}; - } - } - return result; + const std::string path = + std::string(endpoint) + "?date=" + percent_encode_query_value(date) + + "&signature_type=" + std::to_string(static_cast(sig_type_)) + + "&next_cursor=" + percent_encode_query_value(cursor); + return read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); + }, + parse_user_earning_info); } } diff --git a/src/clob_position_pages.cpp b/src/clob_position_pages.cpp index 4ed2a55..59caf1e 100644 --- a/src/clob_position_pages.cpp +++ b/src/clob_position_pages.cpp @@ -1,11 +1,11 @@ #include "clob_positions_internal.hpp" +#include "polymarket/pagination.hpp" #include "query_encoding.hpp" #include #include #include -#include #include #include @@ -129,30 +129,37 @@ namespace polymarket::detail { if (address.empty() || !fetch) return {}; - std::vector result; + constexpr const char *endpoint = "/positions"; try { - const std::string prefix = "/positions?user=" + - percent_encode_query_value(address) + + const std::string prefix = std::string(endpoint) + + "?user=" + percent_encode_query_value(address) + filter_query(filter) + "&limit=500&offset="; - for (size_t offset = 0; offset <= MAX_OFFSET; offset += PAGE_LIMIT) - { - const auto response = fetch(prefix + std::to_string(offset)); - if (!response.ok()) - return {}; - auto page = parse_page(response.body); - const bool terminal = page.size() < PAGE_LIMIT; - result.reserve(result.size() + page.size()); - result.insert(result.end(), std::make_move_iterator(page.begin()), - std::make_move_iterator(page.end())); - if (terminal) - break; - } + auto paginator = Paginator::offset( + endpoint, + [&](const PageRequest &request) -> Result> + { + const auto response = fetch(prefix + std::to_string(request.offset)); + if (!response.ok()) + return Result>::failure(make_sdk_error(response, endpoint)); + try + { + return Result>::success( + Page{parse_page(response.body), ""}); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + PAGE_LIMIT, MAX_OFFSET); + auto positions = paginator.collect(); + return positions ? std::move(positions.value()) : std::vector{}; } catch (...) { return {}; } - return result; } } From 9229e8cd172e0c69f8608d402ee5d33fa26846a9 Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 12:59:57 -0400 Subject: [PATCH 3/6] feat(clob): add public pagers for markets, orders, trades and positions --- cmake/PolymarketTests.cmake | 4 + include/polymarket/clob_client.hpp | 25 +++- src/clob_account.cpp | 51 +++++--- src/clob_client.cpp | 57 +++++++++ src/clob_position_pages.cpp | 67 ++++++---- src/clob_positions.cpp | 27 ++++ src/clob_positions_internal.hpp | 5 + tests/test_clob_pagers.cpp | 194 +++++++++++++++++++++++++++++ 8 files changed, 385 insertions(+), 45 deletions(-) create mode 100644 tests/test_clob_pagers.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index f4f0de1..bceec73 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -179,6 +179,10 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_pagination PRIVATE polymarket::client) add_test(NAME test_pagination COMMAND test_pagination) + add_executable(test_clob_pagers tests/test_clob_pagers.cpp) + target_link_libraries(test_clob_pagers PRIVATE polymarket::client) + add_test(NAME test_clob_pagers COMMAND test_clob_pagers) + add_executable(test_sdk_error tests/test_sdk_error.cpp) target_link_libraries(test_sdk_error PRIVATE polymarket::client) add_test(NAME test_sdk_error COMMAND test_sdk_error) diff --git a/include/polymarket/clob_client.hpp b/include/polymarket/clob_client.hpp index 56f9de7..d4c3d15 100644 --- a/include/polymarket/clob_client.hpp +++ b/include/polymarket/clob_client.hpp @@ -7,6 +7,7 @@ #include "polymarket/types.hpp" #include "polymarket/http_client.hpp" #include "polymarket/order_signer.hpp" +#include "polymarket/pagination.hpp" #include "polymarket/sdk_error.hpp" #include #include @@ -80,6 +81,14 @@ namespace polymarket ClobMarketPage get_simplified_markets(const std::string &next_cursor = ""); ClobMarketPage get_sampling_simplified_markets(const std::string &next_cursor = ""); + // Pagers fetch lazily, starting at `start_cursor` (or the first page), + // and can resume from resume_token(). This client must outlive them. + Paginator markets_pager(const std::string &start_cursor = ""); + Paginator sampling_markets_pager(const std::string &start_cursor = ""); + Paginator simplified_markets_pager(const std::string &start_cursor = ""); + Paginator + sampling_simplified_markets_pager(const std::string &start_cursor = ""); + // Orderbook std::optional get_order_book(const std::string &token_id); std::map get_order_books(const std::vector &token_ids); @@ -186,6 +195,10 @@ namespace polymarket std::vector get_open_orders(const std::string &market = ""); Result> get_open_orders_result(const std::string &market = ""); std::vector get_trades(const std::string &next_cursor = ""); + // Lazy pagers; fail with Auth on the first page when not authenticated. + Paginator open_orders_pager(const std::string &market = "", + const std::string &start_cursor = ""); + Paginator trades_pager(const std::string &start_cursor = ""); // Empty when the account has no trade with this ID (yet). std::optional get_trade(const std::string &trade_id); Result> get_trade_result(const std::string &trade_id); @@ -385,8 +398,16 @@ namespace polymarket // Get positions that can be merged (user holds both Yes and No outcomes) std::vector get_mergeable_positions(const std::string &user_address = ""); - - private: + // Offset pagers over the same queries; truncated() is set at the Data + // API offset cap. Fail with InvalidArgument when no address resolves. + Paginator positions_pager(const std::string &user_address = "", + size_t start_offset = 0); + Paginator redeemable_positions_pager(const std::string &user_address = "", + size_t start_offset = 0); + Paginator mergeable_positions_pager(const std::string &user_address = "", + size_t start_offset = 0); + + private: HttpClient http_; HttpClient order_http_; HttpClient data_http_; diff --git a/src/clob_account.cpp b/src/clob_account.cpp index 74031e3..2d8e2f7 100644 --- a/src/clob_account.cpp +++ b/src/clob_account.cpp @@ -135,21 +135,28 @@ namespace polymarket Result> ClobClient::get_open_orders_result(const std::string &market) { - constexpr const char *endpoint = "/data/orders"; - if (!order_signer_ || !api_creds_) - { - return Result>::failure(make_auth_error("Client not authenticated", endpoint)); - } + return open_orders_pager(market).collect(); + } - auto paginator = Paginator::cursor( + Paginator ClobClient::open_orders_pager(const std::string &market, + const std::string &start_cursor) + { + constexpr const char *endpoint = "/data/orders"; + std::string prefix = endpoint; + prefix += market.empty() ? "?" : "?market=" + percent_encode_query_value(market) + "&"; + prefix += "next_cursor="; + return Paginator::cursor( endpoint, - [&](const PageRequest &request) -> Result> + [this, + prefix = std::move(prefix)](const PageRequest &request) -> Result> { - std::string path = endpoint; - path += - market.empty() ? "?" : "?market=" + percent_encode_query_value(market) + "&"; - path += "next_cursor=" + percent_encode_query_value(request.cursor); + if (!order_signer_ || !api_creds_) + { + return Result>::failure( + make_auth_error("Client not authenticated", endpoint)); + } + const std::string path = prefix + percent_encode_query_value(request.cursor); auto response = read([&] { return http_.get(path, get_l2_headers("GET", endpoint, "")); }); @@ -170,17 +177,27 @@ namespace polymarket make_parse_error(ex.what(), endpoint, response.body)); } }, - INITIAL_CURSOR, END_CURSOR); - return paginator.collect(); + start_cursor.empty() ? INITIAL_CURSOR : start_cursor, END_CURSOR); } std::vector ClobClient::get_trades(const std::string &next_cursor) + { + if (!order_signer_ || !api_creds_) throw std::runtime_error("Client not authenticated"); + auto trades = trades_pager(next_cursor).collect(); + return trades ? std::move(trades.value()) : std::vector{}; + } + + Paginator ClobClient::trades_pager(const std::string &start_cursor) { constexpr const char *endpoint = "/data/trades"; - auto paginator = Paginator::cursor( + return Paginator::cursor( endpoint, - [&](const PageRequest &request) -> Result> + [this](const PageRequest &request) -> Result> { + if (!order_signer_ || !api_creds_) + return Result>::failure( + make_auth_error("Client not authenticated", endpoint)); + const std::string path = std::string(endpoint) + "?next_cursor=" + percent_encode_query_value(request.cursor); auto response = @@ -201,9 +218,7 @@ namespace polymarket make_parse_error(ex.what(), endpoint, response.body)); } }, - next_cursor.empty() ? INITIAL_CURSOR : next_cursor, END_CURSOR); - auto trades = paginator.collect(); - return trades ? std::move(trades.value()) : std::vector{}; + start_cursor.empty() ? INITIAL_CURSOR : start_cursor, END_CURSOR); } std::optional ClobClient::get_balance_allowance( diff --git a/src/clob_client.cpp b/src/clob_client.cpp index a06e19f..51c64e7 100644 --- a/src/clob_client.cpp +++ b/src/clob_client.cpp @@ -95,6 +95,34 @@ namespace polymarket } } + template + static Paginator market_paginator(Fetch fetch, const std::string &endpoint, + const std::string &start_cursor) + { + std::string prefix = endpoint + "?next_cursor="; + return Paginator::cursor( + endpoint, + [fetch = std::move(fetch), endpoint, + prefix = std::move(prefix)](const PageRequest &request) -> Result> + { + const auto response = fetch(prefix + percent_encode_query_value(request.cursor)); + if (!response.ok()) + return Result>::failure(make_sdk_error(response, endpoint)); + try + { + auto page = detail::parse_clob_market_page_json(response.body); + return Result>::success( + Page{std::move(page.data), std::move(page.next_cursor)}); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + start_cursor.empty() ? detail::INITIAL_CURSOR : start_cursor, detail::END_CURSOR); + } + ClobClient::ClobClient(Environment environment, const std::string *private_key, const ApiCredentials *creds, SignatureType sig_type, const std::string &funder_address, @@ -318,4 +346,33 @@ namespace polymarket { return read([&] { return http_.get(path); }); }, "/sampling-simplified-markets", next_cursor); } + + Paginator ClobClient::markets_pager(const std::string &start_cursor) + { + return market_paginator([this](const std::string &path) + { return read([&] { return http_.get(path); }); }, "/markets", + start_cursor); + } + + Paginator ClobClient::sampling_markets_pager(const std::string &start_cursor) + { + return market_paginator([this](const std::string &path) + { return read([&] { return http_.get(path); }); }, + "/sampling-markets", start_cursor); + } + + Paginator ClobClient::simplified_markets_pager(const std::string &start_cursor) + { + return market_paginator([this](const std::string &path) + { return read([&] { return http_.get(path); }); }, + "/simplified-markets", start_cursor); + } + + Paginator + ClobClient::sampling_simplified_markets_pager(const std::string &start_cursor) + { + return market_paginator([this](const std::string &path) + { return read([&] { return http_.get(path); }); }, + "/sampling-simplified-markets", start_cursor); + } } diff --git a/src/clob_position_pages.cpp b/src/clob_position_pages.cpp index 59caf1e..e8ab785 100644 --- a/src/clob_position_pages.cpp +++ b/src/clob_position_pages.cpp @@ -1,6 +1,5 @@ #include "clob_positions_internal.hpp" -#include "polymarket/pagination.hpp" #include "query_encoding.hpp" #include @@ -123,38 +122,56 @@ namespace polymarket::detail } } + Paginator position_paginator(const std::string &address, PositionFilter filter, + PositionFetch fetch, size_t start_offset) + { + constexpr const char *endpoint = "/positions"; + if (address.empty()) + { + SdkError error; + error.code = SdkErrorCode::InvalidArgument; + error.message = "positions require a user address"; + error.endpoint = endpoint; + return Paginator::offset( + endpoint, + [error](const PageRequest &) { return Result>::failure(error); }, + PAGE_LIMIT, MAX_OFFSET, start_offset); + } + if (!fetch) throw std::invalid_argument("position fetch must be set"); + std::string prefix = std::string(endpoint) + + "?user=" + percent_encode_query_value(address) + filter_query(filter) + + "&limit=500&offset="; + return Paginator::offset( + endpoint, + [prefix = std::move(prefix), + fetch = std::move(fetch)](const PageRequest &request) -> Result> + { + const auto response = fetch(prefix + std::to_string(request.offset)); + if (!response.ok()) + return Result>::failure(make_sdk_error(response, endpoint)); + try + { + return Result>::success( + Page{parse_page(response.body), ""}); + } + catch (const std::exception &ex) + { + return Result>::failure( + make_parse_error(ex.what(), endpoint, response.body)); + } + }, + PAGE_LIMIT, MAX_OFFSET, start_offset); + } + std::vector fetch_position_pages(const std::string &address, PositionFilter filter, const PositionFetch &fetch) { if (address.empty() || !fetch) return {}; - constexpr const char *endpoint = "/positions"; try { - const std::string prefix = std::string(endpoint) + - "?user=" + percent_encode_query_value(address) + - filter_query(filter) + "&limit=500&offset="; - auto paginator = Paginator::offset( - endpoint, - [&](const PageRequest &request) -> Result> - { - const auto response = fetch(prefix + std::to_string(request.offset)); - if (!response.ok()) - return Result>::failure(make_sdk_error(response, endpoint)); - try - { - return Result>::success( - Page{parse_page(response.body), ""}); - } - catch (const std::exception &ex) - { - return Result>::failure( - make_parse_error(ex.what(), endpoint, response.body)); - } - }, - PAGE_LIMIT, MAX_OFFSET); - auto positions = paginator.collect(); + auto positions = position_paginator(address, filter, fetch).collect(); return positions ? std::move(positions.value()) : std::vector{}; } catch (...) diff --git a/src/clob_positions.cpp b/src/clob_positions.cpp index 1183dd1..23e8cef 100644 --- a/src/clob_positions.cpp +++ b/src/clob_positions.cpp @@ -40,4 +40,31 @@ namespace polymarket position_address(*this, user_address), detail::PositionFilter::mergeable, [this](const std::string &path) { return read([&] { return data_http_.get(path); }); }); } + + Paginator ClobClient::positions_pager(const std::string &user_address, + size_t start_offset) + { + return detail::position_paginator( + position_address(*this, user_address), detail::PositionFilter::all, + [this](const std::string &path) { return read([&] { return data_http_.get(path); }); }, + start_offset); + } + + Paginator + ClobClient::redeemable_positions_pager(const std::string &user_address, size_t start_offset) + { + return detail::position_paginator( + position_address(*this, user_address), detail::PositionFilter::redeemable, + [this](const std::string &path) { return read([&] { return data_http_.get(path); }); }, + start_offset); + } + + Paginator + ClobClient::mergeable_positions_pager(const std::string &user_address, size_t start_offset) + { + return detail::position_paginator( + position_address(*this, user_address), detail::PositionFilter::mergeable, + [this](const std::string &path) { return read([&] { return data_http_.get(path); }); }, + start_offset); + } } diff --git a/src/clob_positions_internal.hpp b/src/clob_positions_internal.hpp index 107a9da..d47f515 100644 --- a/src/clob_positions_internal.hpp +++ b/src/clob_positions_internal.hpp @@ -2,7 +2,9 @@ #include "polymarket/clob_types.hpp" #include "polymarket/http_client.hpp" +#include "polymarket/pagination.hpp" +#include #include #include #include @@ -18,6 +20,9 @@ namespace polymarket::detail using PositionFetch = std::function; + Paginator position_paginator(const std::string &address, PositionFilter filter, + PositionFetch fetch, size_t start_offset = 0); + std::vector fetch_position_pages(const std::string &address, PositionFilter filter, const PositionFetch &fetch); diff --git a/tests/test_clob_pagers.cpp b/tests/test_clob_pagers.cpp new file mode 100644 index 0000000..345a7ab --- /dev/null +++ b/tests/test_clob_pagers.cpp @@ -0,0 +1,194 @@ +#include "../src/clob_client_test_fixture.hpp" + +#include +#include +#include + +namespace clob_test +{ + namespace + { + std::string market_page(const std::string &condition_id, const std::string &next_cursor) + { + return R"({"next_cursor":")" + next_cursor + R"(","data":[{"condition_id":")" + + condition_id + + R"(","active":true,"closed":false,"tokens":[{"token_id":"yes","outcome":"Yes"},{"token_id":"no","outcome":"No"}]}]})"; + } + + Environment local_environment(const std::string &url) + { + auto environment = Environment::production(); + environment.clob_url = url; + environment.data_url = url; + return environment; + } + + bool test_markets_pager_follows_cursor() + { + LocalServer server; + ClobClient client(server.url(), 137); + server.enqueue(market_page("first", "N+/=")); + server.enqueue(market_page("second", "LTE=")); + auto pager = client.markets_pager(); + std::vector ids; + for (const auto &market : pager.items()) + ids.push_back(market.condition_id); + const auto requests = server.requests(); + return check(ids == std::vector({"first", "second"}), + "markets pager must cover both pages") && + check(requests.size() == 2 && + requests[0].target == "/markets?next_cursor=MA%3D%3D" && + requests[1].target == "/markets?next_cursor=N%2B%2F%3D", + "markets pager must encode each cursor") && + check(pager.done() && !pager.error(), "markets pager must end cleanly"); + } + + bool test_market_pager_variants_and_resume() + { + LocalServer server; + ClobClient client(server.url(), 137); + server.enqueue(market_page("a", "LTE=")); + server.enqueue(market_page("b", "LTE=")); + server.enqueue(market_page("c", "LTE=")); + server.enqueue(market_page("d", "LTE=")); + (void)client.sampling_markets_pager().collect(); + (void)client.simplified_markets_pager().collect(); + (void)client.sampling_simplified_markets_pager().collect(); + auto resumed = client.markets_pager("saved"); + const auto markets = resumed.collect(); + const auto requests = server.requests(); + return check(requests.size() == 4 && + requests[0].target == "/sampling-markets?next_cursor=MA%3D%3D" && + requests[1].target == "/simplified-markets?next_cursor=MA%3D%3D" && + requests[2].target == + "/sampling-simplified-markets?next_cursor=MA%3D%3D", + "market pager variants must use their endpoints") && + check(requests[3].target == "/markets?next_cursor=saved", + "markets pager must start at start_cursor") && + check(markets.ok() && markets.value().size() == 1, + "resumed markets pager must load"); + } + + bool test_markets_pager_reports_http_failure() + { + LocalServer server; + ClobClient client(server.url(), 137); + server.enqueue(market_page("kept", "N+/=")); + server.enqueue(R"({"error":"boom"})", 500); + auto pager = client.markets_pager(); + std::vector ids; + for (const auto &market : pager.items()) + ids.push_back(market.condition_id); + return check(ids == std::vector({"kept"}), + "items before a failure must be yielded") && + check(pager.error() && pager.error()->http_status == 500 && + pager.error()->endpoint == "/markets", + "markets pager must expose the HTTP failure"); + } + + bool test_open_orders_pager_signs_and_resumes() + { + LocalServer server; + auto client = authenticated_client(server.url()); + server.enqueue(R"({"next_cursor":"LTE=","data":[]})"); + const std::string market = "m+/="; + auto pager = client.open_orders_pager(market, "C2"); + const auto orders = pager.collect(); + const auto requests = server.requests(); + return check(orders.ok() && orders.value().empty(), + "open-orders pager must load an empty page") && + check(requests.size() == 1 && + requests[0].target == "/data/orders?market=m%2B%2F%3D&next_cursor=C2", + "open-orders pager must keep the market filter and start cursor") && + check(requests[0].headers.at("poly_signature") == + expected_signature(requests[0], "/data/orders"), + "open-orders pager must sign the endpoint path"); + } + + bool test_authenticated_pagers_require_credentials() + { + LocalServer server; + ClobClient client(server.url(), 137); + auto orders = client.open_orders_pager().next_page(); + auto trades = client.trades_pager().collect(); + bool legacy_threw = false; + try + { + (void)client.get_trades(); + } + catch (const std::runtime_error &) + { + legacy_threw = true; + } + return check(!orders.ok() && orders.error().code == SdkErrorCode::Auth, + "open-orders pager must fail with Auth") && + check(!trades.ok() && trades.error().code == SdkErrorCode::Auth, + "trades pager must fail with Auth") && + check(legacy_threw, "legacy get_trades must still throw when unauthenticated") && + check(server.requests().empty(), + "unauthenticated pagers must not send requests"); + } + + bool test_trades_pager_follows_cursor() + { + LocalServer server; + auto client = authenticated_client(server.url()); + server.enqueue(R"({"next_cursor":"T2","data":[]})"); + server.enqueue(R"({"next_cursor":"LTE=","data":[]})"); + auto pager = client.trades_pager("T1"); + const auto trades = pager.collect(); + const auto requests = server.requests(); + return check(trades.ok(), "trades pager must succeed") && + check(requests.size() == 2 && + requests[0].target == "/data/trades?next_cursor=T1" && + requests[1].target == "/data/trades?next_cursor=T2", + "trades pager must follow cursors from start_cursor") && + check(pager.resume_token().cursor == "LTE=", + "exhausted trades pager must hold the end cursor"); + } + + bool test_positions_pager_offsets_and_address() + { + LocalServer server; + ClobClient client(local_environment(server.url())); + auto missing = client.positions_pager().next_page(); + const bool missing_rejected = !missing.ok() && + missing.error().code == SdkErrorCode::InvalidArgument && + server.requests().empty(); + + server.enqueue(R"([])"); + server.enqueue(R"([])"); + server.enqueue(R"([])"); + const std::string user = "0x1111111111111111111111111111111111111111"; + auto all = client.positions_pager(user, 500).collect(); + (void)client.redeemable_positions_pager(user).collect(); + (void)client.mergeable_positions_pager(user).collect(); + const auto requests = server.requests(); + return check(missing_rejected, + "positions pager must reject a missing address before any request") && + check(all.ok() && all.value().empty(), + "positions pager must load an empty page") && + check(requests.size() == 3 && requests[0].target == "/positions?user=" + user + + "&limit=500&offset=500", + "positions pager must start at start_offset") && + check(requests[1].target == "/positions?user=" + user + + "&redeemable=true&limit=500&offset=0" && + requests[2].target == + "/positions?user=" + user + "&mergeable=true&limit=500&offset=0", + "position pager variants must keep their filters"); + } + } // namespace +} // namespace clob_test + +int main() +{ + using namespace clob_test; + const bool passed = + test_markets_pager_follows_cursor() && test_market_pager_variants_and_resume() && + test_markets_pager_reports_http_failure() && test_open_orders_pager_signs_and_resumes() && + test_authenticated_pagers_require_credentials() && test_trades_pager_follows_cursor() && + test_positions_pager_offsets_and_address(); + if (!passed) return 1; + std::cout << "test_clob_pagers passed\n"; + return 0; +} From 8ba578ca591db616e000bf56ef126b1d8e669cca Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 13:08:51 -0400 Subject: [PATCH 4/6] refactor(market-fetcher): page CLOB markets through Paginator --- include/polymarket/pagination.hpp | 5 +- src/market_fetcher.cpp | 101 +++++++++++++++--------------- tests/test_pagination.cpp | 10 ++- 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/include/polymarket/pagination.hpp b/include/polymarket/pagination.hpp index 0764910..134d817 100644 --- a/include/polymarket/pagination.hpp +++ b/include/polymarket/pagination.hpp @@ -84,7 +84,8 @@ namespace polymarket }; // Pulls pages lazily through `fetch`. Cursor pagination ends at an empty or - // end cursor and fails on a cycle or after max_pages. Offset pagination ends + // end cursor and fails after max_pages or on a cycle; the page whose next + // cursor repeats is still returned and the call after it fails. Offset pagination ends // on a short page and stops with truncated() set once the next offset would // pass `max_offset`; items inserted between requests can shift across pages. // After a failure every call reports the same error. Not thread-safe; any @@ -242,7 +243,7 @@ namespace polymarket } catch (const detail::OpaqueCursorPaginationError &ex) { - return fail(make_parse_error(ex.what(), endpoint_)); + error_ = make_parse_error(ex.what(), endpoint_); } return PageResult::success(std::move(page.value().items)); } diff --git a/src/market_fetcher.cpp b/src/market_fetcher.cpp index 2d1bc7a..be3eba4 100644 --- a/src/market_fetcher.cpp +++ b/src/market_fetcher.cpp @@ -1,11 +1,11 @@ #include "polymarket/market_fetcher.hpp" +#include "polymarket/pagination.hpp" #include "rate_limit_internal.hpp" #include "query_encoding.hpp" #include "rest_orderbook_parsing.hpp" #include #include -#include using json = nlohmann::json; @@ -41,67 +41,64 @@ namespace polymarket std::vector MarketFetcher::fetch_market_pages( std::size_t limit, bool neg_risk_only) { - constexpr std::size_t max_pages = 1000; + constexpr const char *initial_cursor = "MA=="; const std::string endpoint = neg_risk_only ? "/sampling-markets" : "/markets"; const uint64_t current_time_ms = now_sec() * 1000; - std::vector markets; - std::string next_cursor; - std::unordered_set seen_cursors; - for (std::size_t page_number = 0; - page_number < max_pages && markets.size() < limit; - ++page_number) - { - if (!seen_cursors.insert(next_cursor).second) - break; - const std::string path = next_cursor.empty() - ? endpoint - : endpoint + "?next_cursor=" + - detail::percent_encode_query_value(next_cursor); - const auto response = detail::retry_rate_limited(config_.rate_limit_retry, - [&] { return http_.get(path); }); - if (!response.ok()) - { - std::cerr << "Failed to fetch markets: " << response.status_code - << " - " << response.error << '\n'; - break; - } - try + auto paginator = Paginator::cursor( + endpoint, + [&](const PageRequest &request) -> Result> { - auto page = parse_markets_response(response.body); - for (auto &market : page) + const std::string path = + request.cursor == initial_cursor + ? endpoint + : endpoint + + "?next_cursor=" + detail::percent_encode_query_value(request.cursor); + const auto response = detail::retry_rate_limited(config_.rate_limit_retry, + [&] { return http_.get(path); }); + if (!response.ok()) { - const bool valid_neg_risk = market.neg_risk && market.active && - !market.closed && !market.condition_id.empty() && - !market.token_yes().empty() && - !market.token_no().empty() && - market.end_time_ms > current_time_ms; - if (neg_risk_only && !valid_neg_risk) - { - continue; - } - if (markets.size() == limit) - { - break; - } - markets.push_back(std::move(market)); + std::cerr << "Failed to fetch markets: " << response.status_code << " - " + << response.error << '\n'; + return Result>::failure(make_sdk_error(response, endpoint)); } - const auto parsed = json::parse(response.body); - if (!parsed.is_object() || !parsed.contains("next_cursor") || - parsed["next_cursor"].is_null()) + Page page; + try { - break; + page.items = parse_markets_response(response.body); + const auto parsed = json::parse(response.body); + if (parsed.is_object() && parsed.contains("next_cursor") && + !parsed["next_cursor"].is_null()) + page.next_cursor = parsed["next_cursor"].get(); } - const auto cursor = parsed["next_cursor"].get(); - if (cursor.empty() || cursor == next_cursor || cursor == "LTE=") + catch (const std::exception &error) { - break; + std::cerr << "Market response parse error: " << error.what() << '\n'; + page.next_cursor.clear(); } - next_cursor = cursor; - } - catch (const std::exception &error) + return Result>::success(std::move(page)); + }, + initial_cursor, "LTE="); + + std::vector markets; + while (markets.size() < limit) + { + auto page = paginator.next_page(); + if (!page.ok() || !page.value()) break; + for (auto &market : *page.value()) { - std::cerr << "Market response parse error: " << error.what() << '\n'; - break; + const bool valid_neg_risk = + market.neg_risk && market.active && !market.closed && + !market.condition_id.empty() && !market.token_yes().empty() && + !market.token_no().empty() && market.end_time_ms > current_time_ms; + if (neg_risk_only && !valid_neg_risk) + { + continue; + } + if (markets.size() == limit) + { + break; + } + markets.push_back(std::move(market)); } } return markets; diff --git a/tests/test_pagination.cpp b/tests/test_pagination.cpp index cdada3b..45002b8 100644 --- a/tests/test_pagination.cpp +++ b/tests/test_pagination.cpp @@ -95,10 +95,14 @@ namespace check(first.ok() && first.value() && *first.value() == std::vector({1}), "first page must load"); auto second = paginator.next_page(); - check(!second.ok() && second.error().code == SdkErrorCode::Parse, - "cursor cycle must fail as parse"); - expect_equal("cycle endpoint", second.error().endpoint, "/items"); + check(second.ok() && second.value() && *second.value() == std::vector({2}), + "page that repeats a cursor must still be returned"); check(paginator.error().has_value() && paginator.done(), "cycle must set error()"); + auto third = paginator.next_page(); + check(!third.ok() && third.error().code == SdkErrorCode::Parse, + "call after a cursor cycle must fail as parse"); + expect_equal("cycle endpoint", third.error().endpoint, "/items"); + check(fetch.requests.size() == 2, "cursor cycle must not fetch again"); } void test_cursor_page_cap_fails() From 068ce0e8c0702e0826af1c2c423a12a1a81cc73c Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 13:52:13 -0400 Subject: [PATCH 5/6] fix(pagination): reach iterator state through a member for GCC 11 --- include/polymarket/pagination.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/polymarket/pagination.hpp b/include/polymarket/pagination.hpp index 134d817..fe09b9f 100644 --- a/include/polymarket/pagination.hpp +++ b/include/polymarket/pagination.hpp @@ -187,11 +187,16 @@ namespace polymarket friend bool operator==(const ItemIterator &it, std::default_sentinel_t) { - return it.paginator_ == nullptr || - it.paginator_->buffer_index_ >= it.paginator_->buffer_.size(); + return it.at_end(); } private: + bool at_end() const + { + return paginator_ == nullptr || + paginator_->buffer_index_ >= paginator_->buffer_.size(); + } + Paginator *paginator_{nullptr}; }; From 0dac65b4cf9a1db075ec3345970c7f03cc696067 Mon Sep 17 00:00:00 2001 From: yluoc Date: Tue, 13 Oct 2026 14:11:28 -0400 Subject: [PATCH 6/6] fix(pagination): advance offset resume token past the final page --- include/polymarket/pagination.hpp | 15 +++++++++----- tests/test_pagination.cpp | 34 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/include/polymarket/pagination.hpp b/include/polymarket/pagination.hpp index fe09b9f..ddfc072 100644 --- a/include/polymarket/pagination.hpp +++ b/include/polymarket/pagination.hpp @@ -68,7 +68,9 @@ namespace polymarket } // namespace detail // The next page to request. Cursor pagination fills `cursor`; offset - // pagination fills `offset` and `limit`. Pass it back to resume later. + // pagination fills `offset` and `limit`. Pass it back to resume later; an + // exhausted cursor holds the end cursor, and an offset past the cap + // resumes as truncated without fetching. struct PageRequest { std::string cursor; @@ -264,12 +266,15 @@ namespace polymarket " items, more than limit " + std::to_string(limit_), endpoint_)); - if (items.size() < limit_) + const bool short_page = items.size() < limit_; + const bool at_offset_cap = !short_page && max_offset_ - offset_ < limit_; + offset_ = items.size() > std::numeric_limits::max() - offset_ + ? std::numeric_limits::max() + : offset_ + items.size(); + if (short_page) done_ = true; - else if (max_offset_ - offset_ < limit_) + else if (at_offset_cap) done_ = truncated_ = true; - else - offset_ += limit_; return PageResult::success(std::move(items)); } diff --git a/tests/test_pagination.cpp b/tests/test_pagination.cpp index 45002b8..56fa3b8 100644 --- a/tests/test_pagination.cpp +++ b/tests/test_pagination.cpp @@ -202,6 +202,38 @@ namespace "resume token must point past the last fetched page"); } + void test_offset_resume_after_cap_does_not_refetch() + { + ScriptedFetch fetch{{page({1, 2}), page({3, 4})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 2); + (void)paginator.collect(); + const auto token = paginator.resume_token(); + check(token.offset == 4, "resume token after the offset cap must point past the last page"); + + ScriptedFetch resumed_fetch; + auto resumed = + IntPaginator::offset("/items", resumed_fetch.fn(), token.limit, 2, token.offset); + check(drain(resumed).empty() && resumed_fetch.requests.empty() && resumed.truncated(), + "resuming past the offset cap must not fetch the final page again"); + } + + void test_offset_resume_after_short_page_does_not_refetch() + { + ScriptedFetch fetch{{page({1, 2}), page({3})}}; + auto paginator = IntPaginator::offset("/items", fetch.fn(), 2, 10'000); + (void)paginator.collect(); + const auto token = paginator.resume_token(); + check(token.offset == 3, + "resume token after a short page must point at the next unseen item"); + + ScriptedFetch resumed_fetch{{page({})}}; + auto resumed = + IntPaginator::offset("/items", resumed_fetch.fn(), token.limit, 10'000, token.offset); + check(drain(resumed).empty() && resumed_fetch.requests.size() == 1 && + resumed_fetch.requests[0].offset == 3, + "resuming after a short page must not return items already seen"); + } + void test_resume_tokens() { ScriptedFetch cursor_fetch{{page({1}, "B"), page({2}, "LTE=")}}; @@ -256,6 +288,8 @@ int main() test_collect_reports_failure(); test_collect_max_items(); test_resume_tokens(); + test_offset_resume_after_cap_does_not_refetch(); + test_offset_resume_after_short_page_does_not_refetch(); test_range_skips_empty_pages_and_resumes_after_break(); return check_support::finish("test_pagination"); }