Repository navigation
refactor(metrics): phase 1 deprecation of the legacy monitor stack #6988
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package org.tron.common.prometheus; | ||
|
|
||
| import io.prometheus.client.Info; | ||
| import java.util.Map; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Slf4j(topic = "metrics") | ||
| class MetricsInfo { | ||
|
|
||
| private static final Map<String, Info> container = new ConcurrentHashMap<>(); | ||
|
|
||
| static { | ||
| init(MetricKeys.Info.NODE_INFO, "tron node info.", | ||
| MetricLabels.Info.VERSION, MetricLabels.Info.CHAIN_ID); | ||
| } | ||
|
|
||
| private MetricsInfo() { | ||
| throw new IllegalStateException("MetricsInfo"); | ||
| } | ||
|
|
||
| private static void init(String name, String help, String... labels) { | ||
| container.put(name, Info.build() | ||
| .name(name) | ||
| .help(help) | ||
| .labelNames(labels) | ||
| .register()); | ||
| } | ||
|
|
||
| static void set(String key, String... labels) { | ||
| if (Metrics.enabled()) { | ||
| Info info = container.get(key); | ||
| if (info == null) { | ||
| logger.info("{} not exist", key); | ||
| return; | ||
| } | ||
| info.labels(labels); | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,7 +24,9 @@ | |
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.context.annotation.Scope; | ||
| import org.springframework.stereotype.Component; | ||
| import org.tron.common.math.StrictMathWrapper; | ||
| import org.tron.common.overlay.message.Message; | ||
| import org.tron.common.parameter.CommonParameter; | ||
| import org.tron.common.prometheus.MetricKeys; | ||
| import org.tron.common.prometheus.Metrics; | ||
| import org.tron.common.utils.Pair; | ||
|
|
@@ -92,6 +94,25 @@ public class PeerConnection { | |
| @Getter | ||
| private volatile long blockRcvTime; | ||
|
|
||
| /** | ||
| * Sliding factor of the fetch latency Exponentially Weighted Moving Average (EWMA): | ||
| * each new sample contributes | ||
| * 1 / FETCH_LATENCY_EWMA_FACTOR (alpha = 0.1), balancing smoothing against | ||
| * responsiveness, in the same order as TCP's SRTT gain (1/8, RFC 6298). | ||
| * | ||
| * <p>Blend-then-clamp is intentional: samples blend at their measured value and are | ||
| * clamped to fetchBlock.timeout only afterwards, so sustained degradation saturates | ||
| * within 1-3 samples (issue #6923). Accepted cost: one transient outlier (~10x a | ||
| * healthy estimate) can jump the estimate straight to the clamp, needing ~9 healthy | ||
| * samples to decay; hard-timeout fallback and the saturation gate (one secondary | ||
| * fetch per tick) bound the impact meanwhile. | ||
| */ | ||
| private static final int FETCH_LATENCY_EWMA_FACTOR = 10; | ||
|
|
||
| private volatile long fetchLatency; | ||
|
|
||
| private volatile boolean hasFetchLatencySampled; | ||
|
|
||
| @Getter | ||
| @Setter | ||
| private volatile TronState tronState = TronState.INIT; | ||
|
|
@@ -184,6 +205,45 @@ public void setChannel(Channel channel) { | |
| Args.getInstance().getRateLimiterDisconnect()); | ||
| } | ||
|
|
||
| /** | ||
| * Updates the fetch latency estimator, which has an explicit unsampled state: the | ||
| * channel's average latency is only a read fallback and never enters the sample sequence. | ||
| * The first measured sample directly replaces the placeholder; subsequent samples are | ||
| * blended with alpha = 1 / FETCH_LATENCY_EWMA_FACTOR. A single fetch worker reads while | ||
| * the channel event loop writes, so volatile suffices and no lock should be added. | ||
| * Clamping applies only after blending, by design (see {@link #FETCH_LATENCY_EWMA_FACTOR}). | ||
| * | ||
| * @param latencyMillis measured fetch latency in milliseconds | ||
| */ | ||
| public void updateFetchLatency(long latencyMillis) { | ||
|
Collaborator
There was a problem hiding this comment. [DISCUSS] A single transient outlier can saturate a warmed estimate The sample is blended before clamping, so one slow response can push a warmed estimate straight to the clamp. #6923 measured this ( Example with timeout = 500 and a warmed estimate of 100 ms: any single sample ≥ 4100 ms saturates. Samples can be up to
The old P75 over a ~1028-sample decaying reservoir was essentially insensitive to a single outlier. This costs at most some extra secondaries and duplicate downloads, not correctness, so it may well be acceptable. If not, a one-line change caps any single sample's influence at Either way, it would help to state the choice in the Javadoc next to the α rationale.
Contributor
Author
There was a problem hiding this comment. Thanks for looking at this — you're right that the transient-outlier case is not covered by the #6923 runs, and the arithmetic checks out: with the blend applied before the clamp, a single sample of ≥ ~10× a healthy estimate moves the estimate straight to the clamp, and at α = 0.1 it takes about nine healthy samples to decay back (≈27 s at the 3 s block cadence). We kept blend-then-clamp deliberately. Clamping the sample instead of the estimate caps each sample's contribution, so under sustained degradation the estimate would approach the clamp only geometrically — roughly 20+ samples versus the measured 1–3 — which directly weakens the saturation-gate switching that motivated this change, and the convergence behavior pinned in #6923. The transient cost is bounded as you note: while the estimate is saturated, the worst case is one secondary dispatch per 50 ms worker tick, every response still feeds the estimator a real measured sample, hard timeout remains the unconditional backstop, and the ranking self-heals within ~9 samples. So we prefer keeping the current semantics; the choice, its rationale, and the transient-outlier cost are now stated in the estimator javadoc (cb7dd42). Happy to revisit sample-side clamping if you still prefer it. |
||
| if (!hasFetchLatencySampled) { | ||
| fetchLatency = clampFetchLatency(latencyMillis); | ||
| hasFetchLatencySampled = true; | ||
| } else { | ||
| fetchLatency = clampFetchLatency( | ||
| (fetchLatency * (FETCH_LATENCY_EWMA_FACTOR - 1) + latencyMillis) | ||
| / FETCH_LATENCY_EWMA_FACTOR); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Returns the bounded fetch latency estimate. While the estimator has not observed a | ||
| * real fetch sample yet, the channel's average latency is returned as a read fallback | ||
| * (an unknown peer is treated via its transport-level estimate instead of 0). | ||
| */ | ||
| public long getFetchLatency() { | ||
| if (!hasFetchLatencySampled) { | ||
| return channel.getAvgLatency(); | ||
| } | ||
| return fetchLatency; | ||
| } | ||
|
|
||
| private long clampFetchLatency(long latency) { | ||
| // Saturation intentionally makes the >= timeout gate in FetchBlockService trigger. | ||
| return StrictMathWrapper.max(0, | ||
| StrictMathWrapper.min(CommonParameter.getInstance().fetchBlockTimeout, latency)); | ||
| } | ||
|
|
||
| public void setBlockBothHave(BlockId blockId) { | ||
| this.blockBothHave = blockId; | ||
| this.blockBothHaveUpdateTime = System.currentTimeMillis(); | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.