Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rpcdaemon: page_token management in Data API Range ops #2477

Merged
merged 10 commits into from
Nov 7, 2024
Prev Previous commit
Next Next commit
Removed useless functions
Sixtysixter committed Nov 5, 2024
commit e03957b437a43102528418344a678db7aa765b2e
4 changes: 2 additions & 2 deletions silkworm/db/kv/api/endpoint/paginated_sequence.hpp
Original file line number Diff line number Diff line change
@@ -61,7 +61,7 @@ class PaginatedSequence {
Task<std::optional<T>> next() {
if (it_ == current_.values.cend()) {
if (!current_.next_page_token.empty()) {
current_ = co_await next_page_provider_(current_.next_page_token);
current_ = co_await next_page_provider_(std::move(current_.next_page_token));
it_ = current_.values.cbegin();
}
}
@@ -138,7 +138,7 @@ class PaginatedSequencePair {
if (key_it_ == current_.keys.cend()) {
SILKWORM_ASSERT(value_it_ == current_.values.cend());
if (!current_.next_page_token.empty()) {
current_ = co_await next_page_provider_(current_.next_page_token);
current_ = co_await next_page_provider_(std::move(current_.next_page_token));
key_it_ = current_.keys.cbegin();
value_it_ = current_.values.cbegin();
}
35 changes: 7 additions & 28 deletions silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp
Original file line number Diff line number Diff line change
@@ -46,12 +46,12 @@ struct PaginatedSequenceTest : public test_util::ContextTestBase {
struct TestPaginatorUint64 {
explicit TestPaginatorUint64(const PageUint64List& pages) : pages_(pages) {}

Task<PageResultUint64> operator()() {
Task<PageResultUint64> operator()(std::string /*page_token*/) {
if (count_ == 0 && pages_.empty()) {
co_return PageResultUint64{}; // has_more=false as default
co_return PageResultUint64{};
}
if (count_ < pages_.size()) {
const auto next_token = (count_ != pages_.size() - 1) ? "again" : "";
const auto next_token = (count_ != pages_.size() - 1) ? "next" : "";
PageResultUint64 page_result{pages_[count_], next_token};
++count_;
co_return page_result;
@@ -67,11 +67,8 @@ struct TestPaginatorUint64 {
TEST_CASE_METHOD(PaginatedSequenceTest, "paginated_uint64_sequence: empty sequence", "[db][kv][api][paginated_sequence]") {
PageUint64List empty;
TestPaginatorUint64 paginator{empty};
auto lambda = [&](std::string) mutable -> Task<PageResultUint64> {
co_return co_await paginator();
};

PaginatedUint64 paginated{lambda};
PaginatedUint64 paginated{paginator};
// We're using this lambda instead of built-in paginated_to_vector just to check Iterator::has_next
const auto paginated_it_to_vector = [](auto& ps) -> Task<std::vector<uint64_t>> {
auto it = co_await ps.begin();
@@ -92,11 +89,7 @@ TEST_CASE_METHOD(PaginatedSequenceTest, "paginated_uint64_sequence: non-empty se
for (const auto& [page_list, expected_sequence] : fixtures) {
SECTION("test vector: " + std::to_string(++i)) {
TestPaginatorUint64 paginator{page_list};
auto lambda = [&](const std::string&) mutable -> Task<PageResultUint64> {
co_return co_await paginator();
};

PaginatedUint64 paginated{lambda};
PaginatedUint64 paginated{paginator};
CHECK(spawn_and_wait(paginated_to_vector(paginated)) == expected_sequence);
}
}
@@ -190,14 +183,7 @@ TEST_CASE_METHOD(PaginatedSequenceTest, "set_intersection", "[db][kv][api][pagin
SECTION("test vector: " + std::to_string(++i)) {
const auto& [v1, v2] = v1_v2_pair;
TestPaginatorUint64 paginator1{v1}, paginator2{v2};
auto lambda1 = [&](std::string) mutable -> Task<PageResultUint64> {
co_return co_await paginator1();
};
auto lambda2 = [&](std::string) mutable -> Task<PageResultUint64> {
co_return co_await paginator2();
};

PaginatedUint64 paginated1{lambda1}, paginated2{lambda2};
PaginatedUint64 paginated1{paginator1}, paginated2{paginator2};
const auto async_intersection = [&](PaginatedUint64& ps1, PaginatedUint64& ps2) -> Task<std::vector<uint64_t>> {
IntersectionIterator it = set_intersection(co_await ps1.begin(), co_await ps2.begin());
CHECK(co_await it.has_next() == !expected_intersection_set.empty());
@@ -221,14 +207,7 @@ TEST_CASE_METHOD(PaginatedSequenceTest, "set_union", "[db][kv][api][paginated_se
SECTION("test vector: " + std::to_string(++i)) {
const auto& [v1, v2] = v1_v2_pair;
TestPaginatorUint64 paginator1{v1}, paginator2{v2};
auto lambda1 = [&](std::string) mutable -> Task<PageResultUint64> {
co_return co_await paginator1();
};
auto lambda2 = [&](std::string) mutable -> Task<PageResultUint64> {
co_return co_await paginator2();
};

PaginatedUint64 paginated1{lambda1}, paginated2{lambda2};
PaginatedUint64 paginated1{paginator1}, paginated2{paginator2};
const auto async_union = [&](PaginatedUint64& ps1, PaginatedUint64& ps2) -> Task<std::vector<uint64_t>> {
UnionIterator<PaginatedUint64::Iterator> it = set_union(co_await ps1.begin(), co_await ps2.begin());
CHECK(co_await it.has_next() == !expected_union_set.empty());
6 changes: 3 additions & 3 deletions silkworm/db/kv/grpc/client/remote_transaction.cpp
Original file line number Diff line number Diff line change
@@ -151,7 +151,7 @@ Task<api::HistoryPointResult> RemoteTransaction::history_seek(api::HistoryPointQ
Task<api::PaginatedTimestamps> RemoteTransaction::index_range(api::IndexRangeQuery&& query) {
auto paginator = [&, query = std::move(query)](std::string page_token) mutable -> Task<api::PaginatedTimestamps::PageResult> {
query.tx_id = tx_id_;
query.page_token = page_token;
query.page_token = std::move(page_token);
auto request = index_range_request_from_query(query);
try {
const auto reply = co_await rpc::unary_rpc(&Stub::AsyncIndexRange, stub_, std::move(request), grpc_context_);
@@ -169,7 +169,7 @@ Task<api::PaginatedTimestamps> RemoteTransaction::index_range(api::IndexRangeQue
Task<api::PaginatedKeysValues> RemoteTransaction::history_range(api::HistoryRangeQuery&& query) {
auto paginator = [&, query = std::move(query)](std::string page_token) mutable -> Task<api::PaginatedKeysValues::PageResult> {
query.tx_id = tx_id_;
query.page_token = page_token;
query.page_token = std::move(page_token);
auto request = history_range_request_from_query(query);
try {
const auto reply = co_await rpc::unary_rpc(&Stub::AsyncHistoryRange, stub_, std::move(request), grpc_context_);
@@ -187,7 +187,7 @@ Task<api::PaginatedKeysValues> RemoteTransaction::history_range(api::HistoryRang
Task<api::PaginatedKeysValues> RemoteTransaction::domain_range(api::DomainRangeQuery&& query) {
auto paginator = [&, query = std::move(query)](std::string page_token) mutable -> Task<api::PaginatedKeysValues::PageResult> {
query.tx_id = tx_id_;
query.page_token = page_token;
query.page_token = std::move(page_token);
auto request = domain_range_request_from_query(query);
try {
const auto reply = co_await rpc::unary_rpc(&Stub::AsyncDomainRange, stub_, std::move(request), grpc_context_);
2 changes: 1 addition & 1 deletion silkworm/rpc/test_util/dummy_transaction.hpp
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ namespace silkworm::rpc::test {

template <typename Paginated>
inline Paginated empty_paginated_sequence() {
auto paginator = [](const std::string&) -> Task<typename Paginated::PageResult> {
auto paginator = [](std::string) -> Task<typename Paginated::PageResult> {
co_return typename Paginated::PageResult{};
};
return Paginated{paginator};