Skip to content

[refactor](recycler) Track recycler KV metrics inline and remove standalone statistics scan #68005

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wyxxxcat wants to merge 3 commits into
base: master
Choose a base branch
from
+675 −1,530
Open

[refactor](recycler) Track recycler KV metrics inline and remove standalone statistics scan #68005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a16ff6
[refactor](cloud) Refactor recycler metrics collection
wyxxxcat Sep 14, 2026
d06ea09
[refactor](cloud) Remove legacy recycler statistics scanning
wyxxxcat Sep 14, 2026
0510ecc
[test](cloud) Update recycler tests for new metrics
wyxxxcat Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions cloud/src/common/bvars.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,21 +166,33 @@ mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_last_success_ts("recycler_
// resource_id: unique identifier for the repository
// status: status of the recycle task (submitted, completed, error)
mBvarIntAdder g_bvar_recycler_vault_recycle_task_status("recycler_vault_recycle_task_status", {"instance_id", "resource_id", "status"});
// current concurrency of vault delete task
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_num("recycler_instance_last_round_recycled_num", {"instance_id", "resource_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_to_recycle_num("recycler_instance_last_round_to_recycle_num", {"instance_id", "resource_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_bytes("recycler_instance_last_round_recycled_bytes", {"instance_id", "resource_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_to_recycle_bytes("recycler_instance_last_round_to_recycle_bytes", {"instance_id", "resource_type"});
mBvarStatus<double> g_bvar_recycler_instance_last_round_recycle_elpased_ts("recycler_instance_last_round_recycle_elpased_ts", {"instance_id", "resource_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_num(
"recycler_instance_last_round_recycled_num", {"instance_id", "resource_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_bytes(
"recycler_instance_last_round_recycled_bytes", {"instance_id", "resource_type"});
// Elapsed time in milliseconds for the instance's current recycle round.
mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_elapsed_ms(
"recycler_instance_last_round_elapsed_ms", {"instance_id"});

// Current round TxnKV statistics.
mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_scanned(
"instance_recycle_current_round_kv_scanned",
{"instance_id", "task_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_expired(
"instance_recycle_current_round_kv_expired",
{"instance_id", "task_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_recycled(
"instance_recycle_current_round_kv_recycled",
{"instance_id", "task_type"});
mBvarStatus<int64_t> g_bvar_recycler_instance_current_round_recycled_object_bytes(
"instance_current_round_recycled_object_bytes",
{"instance_id", "task_type"});
mBvarStatus<double> g_bvar_recycler_instance_current_round_task_elapsed_ms(
"recycler_instance_current_round_task_elapsed_ms", {"instance_id", "task_type"});
// total recycled num and bytes of resources since recycler started
mBvarInt64Adder g_bvar_recycler_instance_recycle_total_num_since_started("recycler_instance_recycle_total_num_since_started", {"instance_id", "resource_type"});

mBvarInt64Adder g_bvar_recycler_instance_recycle_total_num_since_started("recycler_instance_recycle_total_num_since_started", {"instance_id", "resource_type"});
mBvarInt64Adder g_bvar_recycler_instance_recycle_total_bytes_since_started("recycler_instance_recycle_total_bytes_since_started", {"instance_id", "resource_type"});
mBvarIntAdder g_bvar_recycler_instance_recycle_round("recycler_instance_recycle_round", {"instance_id", "resource_type"});
// represents the ms required per resource to be recycled
// value of -1 means no resource recycled
mBvarStatus<double> g_bvar_recycler_instance_recycle_time_per_resource("recycler_instance_recycle_time_per_resource", {"instance_id", "resource_type"});
// represents the bytes of resources that can be recycled per ms
mBvarStatus<double> g_bvar_recycler_instance_recycle_bytes_per_ms("recycler_instance_recycle_bytes_per_ms", {"instance_id", "resource_type"});
BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_num("recycler",
"packed_file_recycled_kv_num");
BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_bytes(
Expand Down
16 changes: 9 additions & 7 deletions cloud/src/common/bvars.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -714,25 +714,27 @@ extern BvarStatusWithTag<int64_t> g_bvar_recycler_recycle_expired_txn_label_earl
extern BvarStatusWithTag<int64_t> g_bvar_recycler_recycle_restore_job_earlest_ts;

// recycler's mbvars
// instance
extern bvar::Status<int64_t> g_bvar_recycler_task_max_concurrency;
extern mBvarIntAdder g_bvar_recycler_instance_recycle_task_status;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycle_duration;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_next_ts;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_start_ts;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_end_ts;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_last_success_ts;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_elapsed_ms;

// instance, task_type
extern mBvarIntAdder g_bvar_recycler_vault_recycle_task_status;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_num;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_to_recycle_num;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_recycled_bytes;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_last_round_to_recycle_bytes;
extern mBvarStatus<double> g_bvar_recycler_instance_last_round_recycle_elpased_ts;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_scanned;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_expired;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_recycle_current_round_kv_recycled;
extern mBvarStatus<int64_t> g_bvar_recycler_instance_current_round_recycled_object_bytes;
extern mBvarStatus<double> g_bvar_recycler_instance_current_round_task_elapsed_ms;
extern mBvarInt64Adder g_bvar_recycler_instance_recycle_total_num_since_started;
extern mBvarInt64Adder g_bvar_recycler_instance_recycle_total_bytes_since_started;
extern mBvarIntAdder g_bvar_recycler_instance_recycle_round;
extern mBvarStatus<double> g_bvar_recycler_instance_recycle_time_per_resource;
extern mBvarStatus<double> g_bvar_recycler_instance_recycle_bytes_per_ms;
extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_num;
extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_bytes;
extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycle_cost_ms;
Expand All @@ -746,7 +748,7 @@ extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_bytes_object_delet
extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_rowset_scanned_num;

// Operation Log Recycler BVars
// Note: generic metrics (last_round_to_recycle_num/bytes, last_round_recycled_num/bytes, etc.)
// Note: generic metrics (last_round_recycled_num/bytes, etc.)
// are reported by RecyclerMetricsContext with operation_type = "recycle_operation_logs".
extern mBvarStatus<int64_t> g_bvar_recycler_oplog_last_round_total_num;
extern mBvarStatus<int64_t> g_bvar_recycler_oplog_last_round_not_recycled_num;
Expand Down
4 changes: 0 additions & 4 deletions cloud/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,6 @@ CONF_mInt32(instance_recycler_worker_pool_size, "32");
// Max number of delete tasks per batch when recycling objects.
// Each task deletes up to 1000 files. Controls memory usage during large-scale deletion.
CONF_Int32(recycler_max_tasks_per_batch, "1000");
// The worker pool size for http api `statistics_recycle` worker pool
CONF_mInt32(instance_recycler_statistics_recycle_worker_pool_size, "5");
CONF_Bool(enable_checker, "false");
// The parallelism for parallel recycle operation
// s3_producer_pool recycle_tablet_pool, delete single object in this pool
Expand All @@ -130,8 +128,6 @@ CONF_mInt64(delete_bitmap_storage_optimize_v2_check_skip_seconds, "300"); // 5mi
CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min
// interval for check object
CONF_mInt32(check_object_interval_seconds, "43200"); // 12hours
// enable recycler metrics statistics
CONF_Bool(enable_recycler_stats_metrics, "false");
CONF_mBool(retain_deleted_instance_tombstone, "true");

CONF_mInt64(check_recycle_task_interval_seconds, "600"); // 10min
Expand Down
20 changes: 0 additions & 20 deletions cloud/src/common/http_helper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,12 +383,6 @@ const std::unordered_map<std::string_view, HttpHandlerInfo>& get_http_handlers()
return process_recycle_instance((RS*)s, c);
},
.role = HttpRole::RECYCLER}},
{"statistics_recycle",
{.handler =
[](void* s, brpc::Controller* c) {
return process_statistics_recycle((RS*)s, c);
},
.role = HttpRole::RECYCLER}},
{"recycle_copy_jobs",
{.handler =
[](void* s, brpc::Controller* c) {
Expand Down Expand Up @@ -775,20 +769,6 @@ HttpResponse process_recycle_instance(RecyclerServiceImpl* service, brpc::Contro
return http_text_reply(res.status(), res.status().msg());
}

HttpResponse process_statistics_recycle(RecyclerServiceImpl* service, brpc::Controller* cntl) {
std::string request_body = cntl->request_attachment().to_string();
StatisticsRecycleRequest req;
auto st = google::protobuf::util::JsonStringToMessage(request_body, &req);
if (!st.ok()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
"failed to parse StatisticsRecycleRequest");
}
MetaServiceCode code = MetaServiceCode::OK;
std::string msg = "OK";
service->statistics_recycle(req, code, msg);
return http_text_reply(code, msg, msg);
}

HttpResponse process_recycle_copy_jobs(RecyclerServiceImpl* service, brpc::Controller* cntl) {
const auto* instance_id = cntl->http_request().uri().GetQuery("instance_id");
if (!instance_id || instance_id->empty()) {
Expand Down
3 changes: 0 additions & 3 deletions cloud/src/common/http_helper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,6 @@ const std::unordered_map<std::string_view, HttpHandlerInfo>& get_http_handlers()
[[maybe_unused]] HttpResponse process_recycle_instance(RecyclerServiceImpl* service,
brpc::Controller* cntl);

[[maybe_unused]] HttpResponse process_statistics_recycle(RecyclerServiceImpl* service,
brpc::Controller* cntl);

[[maybe_unused]] HttpResponse process_recycle_copy_jobs(RecyclerServiceImpl* service,
brpc::Controller* cntl);

Expand Down
Loading
Loading