diff --git a/.golangci.yml b/.golangci.yml index 1531aa24b..2c0d10e3a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,7 +6,7 @@ linters: - depguard - errorlint - exhaustive - - gomodguard + - gomodguard_v2 - gosec - loggercheck - misspell @@ -88,9 +88,8 @@ linters: errorf: false exhaustive: default-signifies-exhaustive: true - gomodguard: - blocked: - local-replace-directives: false + gomodguard_v2: + local-replace-directives: false gosec: excludes: - G101 diff --git a/libs/chainconsensus/oracle/reporting_plugin.go b/libs/chainconsensus/oracle/reporting_plugin.go index 00ebf2349..db4a6bb26 100644 --- a/libs/chainconsensus/oracle/reporting_plugin.go +++ b/libs/chainconsensus/oracle/reporting_plugin.go @@ -27,6 +27,8 @@ import ( ctypes "github.com/smartcontractkit/capabilities/libs/chainconsensus/types" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" ) const ( @@ -63,11 +65,12 @@ func (c Config) matchingThreshold() int { } type reportingPlugin struct { - config Config - logger logger.SugaredLogger - blocksProvider BlocksProvider - requestsStore RequestsHandler - metrics metrics.ConsensusMetrics + config Config + logger logger.SugaredLogger + blocksProvider BlocksProvider + requestsStore RequestsHandler + metrics metrics.ConsensusMetrics + enableMissingRequestRecovery limits.GateLimiter } func newReportingPlugin( @@ -76,14 +79,22 @@ func newReportingPlugin( blocksProvider BlocksProvider, requestsStore RequestsHandler, metrics metrics.ConsensusMetrics, -) *reportingPlugin { - return &reportingPlugin{ + limitsFactory limits.Factory, +) (*reportingPlugin, error) { + rp := &reportingPlugin{ config: config, logger: logger, blocksProvider: blocksProvider, requestsStore: requestsStore, metrics: metrics, } + + var err error + rp.enableMissingRequestRecovery, err = limits.MakeGateLimiter(limitsFactory, cresettings.Default.MissingRequestRecoveryEnabled) + if err != nil { + return nil, fmt.Errorf("failed to create missing request recovery gate limiter: %w", err) + } + return rp, nil } func (rp *reportingPlugin) Query(ctx context.Context, outctx ocr3types.OutcomeContext) (types.Query, error) { @@ -185,6 +196,12 @@ func (rp *reportingPlugin) Observation( return nil, fmt.Errorf("failed to add observations for missing requests from previous outcome: %w", err) } + open, err := rp.enableMissingRequestRecovery.IsOpen(ctx) + if err != nil { + rp.logger.Errorw("error checking if enableMissingRequestRecovery is enabled", "error", err) + } + observation.EnableMissingRequestRecovery = open + // add observations for requests provided by the leader in the query err = rp.addObservations(ctx, query.RequestIDs, observation) if err != nil { @@ -294,6 +311,11 @@ func (rp *reportingPlugin) getMissingRequestIDs(roundRequests map[string]struct{ } } + if len(missingRequestIDs) > 0 { + rp.logger.Infow("Proposing missing request IDs: present in local store but absent from the leader's query", + "missingRequestIDs", missingRequestIDs) + } + return missingRequestIDs, nil } @@ -563,9 +585,34 @@ func (rp *reportingPlugin) agreeOnMissingRequestIDs(aos []attributedObservation) } sort.Strings(result) + if len(result) > 0 { + rp.logger.Infow("Quorum agreed on missing request IDs: committing them to the outcome for recovery in the next round", + "missingRequestIDs", result) + } return result, nil } +// agreeOnFeatureEnableMissingRequestRecovery reaches DON-wide quorum, once per round, on +// whether missing-request recovery is enabled. This is a per-node, per-round capability flag +// unrelated to any specific request ID, so it is voted on independently of per-request +// observation quorum (which the aggregation loop in Outcome enforces on its own). +// TODO: clean up settings flag and logic when all nodes in all envs support missing-request recovery: https://smartcontract-it.atlassian.net/browse/PLEX-3461 +func (rp *reportingPlugin) agreeOnEnableMissingRequestRecovery(aos []attributedObservation) bool { + minMatching := byzQuorumSize(rp.config.N, rp.config.F) + counter := 0 + for _, ob := range aos { + if ob.Observation.EnableMissingRequestRecovery { + counter++ + if counter >= minMatching { + rp.logger.Infow("Quorum reached for enabling missing request recovery: recovered requests will be aggregated into this round's outcome", + "votes", counter, "required", minMatching) + return true + } + } + } + return false +} + func (rp *reportingPlugin) agreeOnEventuallyConsistentValue(requestID string, aos []attributedObservation) ([]byte, int, error) { iterator := func(yield func(commontypes.OracleID, *observation[[32]byte, []byte]) bool) { for _, ob := range aos { @@ -786,7 +833,34 @@ func (rp *reportingPlugin) Outcome( return nil, fmt.Errorf("failed to unmarshal request IDs: %w", err) } - for _, requestID := range query.RequestIDs { + requestIDs := query.RequestIDs + if prevOutcome := rp.tryUnmarshalPreviousOutcome(outctx); prevOutcome != nil && rp.agreeOnEnableMissingRequestRecovery(aos) { + seen := make(map[string]struct{}, len(query.RequestIDs)) + for _, requestID := range query.RequestIDs { + seen[requestID] = struct{}{} + } + + // Requests that the leader's query omitted but that a quorum of nodes still supplied + // observations for (via addObservationsOfPrevMissingRequests) must still be aggregated here, + // otherwise they would be recycled into MissingRequestIDs forever instead of getting resolved. + recoveredRequestIDs := make([]string, 0, len(prevOutcome.MissingRequestIDs)) + for _, requestID := range prevOutcome.MissingRequestIDs { + if _, ok := seen[requestID]; ok { + continue + } + seen[requestID] = struct{}{} + requestIDs = append(requestIDs, requestID) + recoveredRequestIDs = append(recoveredRequestIDs, requestID) + } + + if len(recoveredRequestIDs) > 0 { + rp.logger.Infow("Missing request recovery: adding previously missing requests to this round's outcome "+ + "(they were omitted by the leader's query but agreed upon by a quorum in the previous round)", + "recoveredRequestIDs", recoveredRequestIDs) + } + } + + for _, requestID := range requestIDs { observationType, err := rp.agreeOnObservationType(requestID, aos) if err != nil { rp.logger.Infow("Could not determine observation type", "requestID", requestID, "err", err) @@ -969,5 +1043,5 @@ func (rp *reportingPlugin) ShouldTransmitAcceptedReport( } func (rp *reportingPlugin) Close() error { - return nil + return rp.enableMissingRequestRecovery.Close() } diff --git a/libs/chainconsensus/oracle/reporting_plugin_factory.go b/libs/chainconsensus/oracle/reporting_plugin_factory.go index 8ff3a42ae..bafcc8591 100644 --- a/libs/chainconsensus/oracle/reporting_plugin_factory.go +++ b/libs/chainconsensus/oracle/reporting_plugin_factory.go @@ -8,6 +8,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/capabilities/libs/chainconsensus/metrics" @@ -21,6 +22,7 @@ type ReportingPluginFactory struct { requestsStore RequestsHandler blocksProvider BlocksProvider metrics metrics.ConsensusMetrics + limitsFactory limits.Factory } func NewReportingPluginFactory( @@ -28,12 +30,14 @@ func NewReportingPluginFactory( requestsStore RequestsHandler, blocksProvider BlocksProvider, metrics metrics.ConsensusMetrics, + limitsFactory limits.Factory, ) *ReportingPluginFactory { return &ReportingPluginFactory{ logger: logger, requestsStore: requestsStore, blocksProvider: blocksProvider, metrics: metrics, + limitsFactory: limitsFactory, } } @@ -69,7 +73,12 @@ func (rpf *ReportingPluginFactory) NewReportingPlugin( pluginLogger := rpf.logger.Named("ChainReadReportingPlugin") - return newReportingPlugin(cfg, pluginLogger, rpf.blocksProvider, rpf.requestsStore, rpf.metrics), ocr3types.ReportingPluginInfo{ + plugin, err := newReportingPlugin(cfg, pluginLogger, rpf.blocksProvider, rpf.requestsStore, rpf.metrics, rpf.limitsFactory) + if err != nil { + return nil, ocr3types.ReportingPluginInfo{}, fmt.Errorf("failed to create reporting plugin: %w", err) + } + + return plugin, ocr3types.ReportingPluginInfo{ Name: rpf.logger.Name() + ".chain-reads-oracle", Limits: ocr3types.ReportingPluginLimits{ MaxQueryLength: int(offchainCfg.MaxQueryLengthBytes), diff --git a/libs/chainconsensus/oracle/reporting_plugin_test.go b/libs/chainconsensus/oracle/reporting_plugin_test.go index 4af4360fb..8e62495e6 100644 --- a/libs/chainconsensus/oracle/reporting_plugin_test.go +++ b/libs/chainconsensus/oracle/reporting_plugin_test.go @@ -25,6 +25,7 @@ import ( commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcapocr3types "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/consensus/ocr3/types" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" @@ -93,13 +94,15 @@ func mustQuery(t *testing.T, requestIDs []string) ocrtypes.Query { func TestObservation(t *testing.T) { t.Run("Error if query is invalid", func(t *testing.T) { - plugin := newReportingPlugin(Config{}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) - _, err := plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, []byte("invalid json")) + plugin, err := newReportingPlugin(Config{}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) + _, err = plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, []byte("invalid json")) require.ErrorContains(t, err, "failed to unmarshal request IDs: proto") }) t.Run("Error if query exceeds max batch size", func(t *testing.T) { - plugin := newReportingPlugin(Config{MaxBatchSize: 2}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) - _, err := plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, mustQuery(t, []string{"1", "2", "3"})) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 2}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) + _, err = plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, mustQuery(t, []string{"1", "2", "3"})) require.EqualError(t, err, "too many request IDs: got 3, expected 2") }) newBlockProvider := func(t *testing.T, chainHeight *types.ChainHeight) *mocks.BlocksProvider { @@ -117,7 +120,8 @@ func TestObservation(t *testing.T) { }) requestsStore := mocks.NewRequestsHandler(t) requestsStore.EXPECT().GetRequestIDs(mock.Anything).Return(nil, nil).Once() - plugin := newReportingPlugin(Config{}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) previousOutcome := &types.Outcome{ ChainHeight: &types.ChainHeight{ Latest: 15, @@ -149,7 +153,8 @@ func TestObservation(t *testing.T) { requestsStore := mocks.NewRequestsHandler(t) requestsStore.EXPECT().GetRequestIDs(mock.Anything).Return(nil, nil).Once() lggr, observedLogs := logger.TestObservedSugared(t, zapcore.ErrorLevel) - plugin := newReportingPlugin(Config{MaxBatchSize: 1}, lggr, blocksProvider, requestsStore, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 1}, lggr, blocksProvider, requestsStore, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) req := tmocks.NewRequest(t) req.EXPECT().GetOCRObservation().Return(nil, errors.New("ocr observation error")).Once() requestsStore.EXPECT().GetRequest("1").Return(req, true) @@ -213,7 +218,8 @@ func TestObservation(t *testing.T) { }) requestsStore.EXPECT().GetRequest(lockableHashableID).Return(lockableHashable, true).Once() - plugin := newReportingPlugin(Config{MaxBatchSize: 50, MaxObservationLength: 1000}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 50, MaxObservationLength: 1000}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) query := mustQuery(t, []string{"request_not_present_in_store", "request_without_observation", "request_with_observation", "lockable_request", "aggregatable_request", "aggregatable_request_without_observation", id, lockableHashableID}) rawObservation, err := plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, query) require.NoError(t, err) @@ -272,7 +278,8 @@ func TestObservation(t *testing.T) { addRequestWithObservation("large_request", 400) - plugin := newReportingPlugin(Config{MaxBatchSize: 50, MaxObservationLength: maxObservationLength}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 50, MaxObservationLength: maxObservationLength}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) query := mustQuery(t, []string{"request_1", "request_2", "large_request", "aggregatable_request"}) rawObservation, err := plugin.Observation(t.Context(), ocr3types.OutcomeContext{}, query) require.NoError(t, err) @@ -327,7 +334,8 @@ func TestObservation(t *testing.T) { "regular_request_won't_fit", }, nil).Once() - plugin := newReportingPlugin(Config{MaxBatchSize: 2, MaxObservationLength: 1000}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 2, MaxObservationLength: 1000}, logger.Sugared(logger.Test(t)), blocksProvider, requestsStore, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) query := mustQuery(t, []string{"regular_request_with_observation", "regular_request_won't_fit"}) previousOutcome := mustMarshalProto(&types.Outcome{ ChainHeight: expectedChainHeight, @@ -633,9 +641,10 @@ func TestValidateObservation(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { lggr := logger.Sugared(logger.Test(t)) - plugin := newReportingPlugin(Config{MaxBatchSize: 10}, lggr, nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{MaxBatchSize: 10}, lggr, nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) - err := plugin.ValidateObservation(t.Context(), tc.outcomeContext, nil, tc.observations) + err = plugin.ValidateObservation(t.Context(), tc.outcomeContext, nil, tc.observations) if tc.expectedError == "" { require.NoError(t, err) } else { @@ -676,7 +685,8 @@ func TestAgreeOnChainHeight(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) aos := make([]attributedObservation, len(tc.observedChainHeights)) for i, chainHeight := range tc.observedChainHeights { aos[i] = attributedObservation{Observation: &types.Observation{ChainHeight: chainHeight}} @@ -706,6 +716,7 @@ func TestOutcome(t *testing.T) { testCases := []struct { name string requestIDs []string + previousOutcome *types.Outcome nodesObservations []types.Observation expectedError string expectedOutcome *types.Outcome @@ -817,6 +828,101 @@ func TestOutcome(t *testing.T) { MissingRequestIDs: []string{"request_1", "request_2"}, }, }, + { + // This reproduces the recovery path added in Observation() via addObservationsOfPrevMissingRequests: + // a request that was flagged as missing in the previous round's Outcome is not present in this + // round's leader query, but a quorum of nodes still supplied observations for it (because they + // proactively re-added it). Outcome() should aggregate it into outcome.Outcomes instead of just + // re-flagging it as missing forever. + name: "recovers observations for previously missing requests not in current query", + requestIDs: []string{"request_in_query"}, + previousOutcome: &types.Outcome{ + ChainHeight: chainHeight, + MissingRequestIDs: []string{"request_missing"}, + }, + nodesObservations: []types.Observation{ + { + // node1 + EnableMissingRequestRecovery: true, + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + { + // node2 + EnableMissingRequestRecovery: true, + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + { + // node3 + EnableMissingRequestRecovery: true, + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + }, + expectedOutcome: &types.Outcome{ + ChainHeight: chainHeight, + Outcomes: []*types.RequestOutcome{ + { + RequestID: "request_in_query", + Outcome: &types.RequestOutcome_EventuallyConsistent{EventuallyConsistent: []byte("value1")}, + }, + { + RequestID: "request_missing", + Outcome: &types.RequestOutcome_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}, + }, + }, + }, + }, + { + // Same setup as the recovery test above, but with the flag left at its default (false). + // Requests missing from the leader's query must NOT be recovered, so behavior stays + // unchanged for nodes that haven't opted in via offchain config yet. + name: "does not recover previously missing requests when flag is disabled", + requestIDs: []string{"request_in_query"}, + previousOutcome: &types.Outcome{ + ChainHeight: chainHeight, + MissingRequestIDs: []string{"request_missing"}, + }, + nodesObservations: []types.Observation{ + { + // node1 + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + { + // node2 + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + { + // node3 + Observations: map[string]*types.RequestObservation{ + "request_in_query": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("value1")}}, + "request_missing": {Observation: &types.RequestObservation_EventuallyConsistent{EventuallyConsistent: []byte("recovered")}}, + }, + }, + }, + expectedOutcome: &types.Outcome{ + ChainHeight: chainHeight, + Outcomes: []*types.RequestOutcome{ + { + RequestID: "request_in_query", + Outcome: &types.RequestOutcome_EventuallyConsistent{EventuallyConsistent: []byte("value1")}, + }, + }, + }, + }, { name: "F+1 nodes agree on request value and F observed error", requestIDs: []string{"request"}, @@ -916,7 +1022,8 @@ func TestOutcome(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { lggr, observed := logger.TestObserved(t, zapcore.DebugLevel) - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(lggr), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(lggr), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) var rawAOs []ocrtypes.AttributedObservation for i := range tc.nodesObservations { nodesObservations := &tc.nodesObservations[i] @@ -925,7 +1032,11 @@ func TestOutcome(t *testing.T) { require.NoError(t, err) rawAOs = append(rawAOs, ocrtypes.AttributedObservation{Observation: rawObservation}) } - rawOutcome, err := plugin.Outcome(t.Context(), ocr3types.OutcomeContext{}, mustQuery(t, tc.requestIDs), rawAOs) + var outcomeContext ocr3types.OutcomeContext + if tc.previousOutcome != nil { + outcomeContext.PreviousOutcome = mustMarshalProto(tc.previousOutcome) + } + rawOutcome, err := plugin.Outcome(t.Context(), outcomeContext, mustQuery(t, tc.requestIDs), rawAOs) if tc.expectedError == "" { require.NoError(t, err) var outcome types.Outcome @@ -996,7 +1107,8 @@ func TestAgreeOnEventuallyConsistentValue(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) nodesObservations := make([]attributedObservation, 0, len(tc.nodesObservations)) for i, ob := range tc.nodesObservations { nodesObservations = append(nodesObservations, attributedObservation{ @@ -1094,7 +1206,8 @@ func TestAgreeOnHashableValue(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) nodesObservations := make([]attributedObservation, 0, len(tc.nodesObservations)) for i, h := range tc.nodesObservations { var ro *types.RequestObservation @@ -1270,7 +1383,8 @@ func TestAgreeOnVolatileValue(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) nodesObservations := make([]attributedObservation, 0, len(tc.observations)) for i := range tc.observations { ro := tc.observations[i] @@ -1391,7 +1505,8 @@ func TestAgreeOnObservationType(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) nodesObservations := make([]attributedObservation, 0, len(tc.observations)) for i := range tc.observations { ob := &tc.observations[i] @@ -1498,7 +1613,8 @@ func TestAggregateValue(t *testing.T) { for _, tc := range testCases { const id = "id" t.Run(tc.name, func(t *testing.T) { - plugin := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + plugin, err := newReportingPlugin(Config{F: 1, N: 4}, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) nodesObservations := make([]attributedObservation, 0, len(tc.observations)) for i := range tc.observations { ob := tc.observations[i] @@ -1698,7 +1814,8 @@ func TestReports(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { lggr, observer := logger.TestObservedSugared(t, zapcore.InfoLevel) - rp := newReportingPlugin(Config{MaxReportCount: tc.maxReportCount, MaxReportLengthBytes: tc.maxReportLengthBytes}, lggr, nil, nil, test.GetConsensusMetrics(t)) + rp, err := newReportingPlugin(Config{MaxReportCount: tc.maxReportCount, MaxReportLengthBytes: tc.maxReportLengthBytes}, lggr, nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) reports, err := rp.Reports(t.Context(), 1, mustMarshalProto(tc.outcome)) require.NoError(t, err) @@ -1822,10 +1939,11 @@ func TestIsVolatileCandidateBetter(t *testing.T) { func TestAgreeOnEventuallyConsistentValue_TwoFPlusOne(t *testing.T) { const id = "req-2f1" // N=7, F=2 → require 5 identical (2F+1) - plugin := newReportingPlugin(Config{ + plugin, err := newReportingPlugin(Config{ F: 2, N: 7, MinResponsesToAggregate: 5, - }, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + }, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) makeAos := func(values []string) []attributedObservation { aos := make([]attributedObservation, len(values)) @@ -1859,9 +1977,10 @@ func TestAgreeOnEventuallyConsistentValue_TwoFPlusOne(t *testing.T) { }) t.Run("default F+1 still works when MinIdenticalObservations is zero", func(t *testing.T) { - pluginDefault := newReportingPlugin(Config{ + pluginDefault, err := newReportingPlugin(Config{ F: 2, N: 7, - }, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t)) + }, logger.Sugared(logger.Test(t)), nil, nil, test.GetConsensusMetrics(t), limits.Factory{}) + require.NoError(t, err) // 3 of 7 matching → F+1=3, should succeed aos := makeAos([]string{"v", "v", "v", "a", "b", "c", "d"}) got, actualCount, err := pluginDefault.agreeOnEventuallyConsistentValue(id, aos) @@ -1890,6 +2009,7 @@ func TestNewReportingPlugin_MinResponsesToAggregateValidation(t *testing.T) { mocks.NewRequestsHandler(t), mocks.NewBlocksProvider(t), test.GetConsensusMetrics(t), + limits.Factory{}, ) } diff --git a/libs/chainconsensus/types/ocr.pb.go b/libs/chainconsensus/types/ocr.pb.go index f3a576177..30fbff27e 100644 --- a/libs/chainconsensus/types/ocr.pb.go +++ b/libs/chainconsensus/types/ocr.pb.go @@ -495,12 +495,13 @@ func (*RequestObservation_Hashable) isRequestObservation_Observation() {} func (*RequestObservation_Volatile) isRequestObservation_Observation() {} type Observation struct { - state protoimpl.MessageState `protogen:"open.v1"` - ChainHeight *ChainHeight `protobuf:"bytes,1,opt,name=chainHeight,proto3" json:"chainHeight,omitempty"` - Observations map[string]*RequestObservation `protobuf:"bytes,2,rep,name=observations,proto3" json:"observations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - MissingRequestIDs []string `protobuf:"bytes,3,rep,name=missingRequestIDs,proto3" json:"missingRequestIDs,omitempty"` // list of requests IDs that are known by observer, but not requested by leader - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + ChainHeight *ChainHeight `protobuf:"bytes,1,opt,name=chainHeight,proto3" json:"chainHeight,omitempty"` + Observations map[string]*RequestObservation `protobuf:"bytes,2,rep,name=observations,proto3" json:"observations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + MissingRequestIDs []string `protobuf:"bytes,3,rep,name=missingRequestIDs,proto3" json:"missingRequestIDs,omitempty"` // list of requests IDs that are known by observer, but not requested by leader + EnableMissingRequestRecovery bool `protobuf:"varint,4,opt,name=enableMissingRequestRecovery,proto3" json:"enableMissingRequestRecovery,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Observation) Reset() { @@ -554,6 +555,13 @@ func (x *Observation) GetMissingRequestIDs() []string { return nil } +func (x *Observation) GetEnableMissingRequestRecovery() bool { + if x != nil { + return x.EnableMissingRequestRecovery + } + return false +} + type RequestError struct { state protoimpl.MessageState `protogen:"open.v1"` Errors [][]byte `protobuf:"bytes,1,rep,name=errors,proto3" json:"errors,omitempty"` @@ -947,11 +955,12 @@ const file_ocr_proto_rawDesc = "" + "\x05error\x18\x04 \x01(\fH\x00R\x05error\x12\x1c\n" + "\bhashable\x18\x05 \x01(\fH\x00R\bhashable\x12P\n" + "\bvolatile\x18\x06 \x01(\v22.chain_capabilities.evm.types.VolatileObservationsH\x00R\bvolatileB\r\n" + - "\vobservation\"\xdc\x02\n" + + "\vobservation\"\xa0\x03\n" + "\vObservation\x12K\n" + "\vchainHeight\x18\x01 \x01(\v2).chain_capabilities.evm.types.ChainHeightR\vchainHeight\x12_\n" + "\fobservations\x18\x02 \x03(\v2;.chain_capabilities.evm.types.Observation.ObservationsEntryR\fobservations\x12,\n" + - "\x11missingRequestIDs\x18\x03 \x03(\tR\x11missingRequestIDs\x1aq\n" + + "\x11missingRequestIDs\x18\x03 \x03(\tR\x11missingRequestIDs\x12B\n" + + "\x1cenableMissingRequestRecovery\x18\x04 \x01(\bR\x1cenableMissingRequestRecovery\x1aq\n" + "\x11ObservationsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12F\n" + "\x05value\x18\x02 \x01(\v20.chain_capabilities.evm.types.RequestObservationR\x05value:\x028\x01\"&\n" + diff --git a/libs/chainconsensus/types/ocr.proto b/libs/chainconsensus/types/ocr.proto index 90f9eff8e..bd899b4ad 100644 --- a/libs/chainconsensus/types/ocr.proto +++ b/libs/chainconsensus/types/ocr.proto @@ -61,6 +61,7 @@ message Observation { ChainHeight chainHeight = 1; map observations = 2; repeated string missingRequestIDs = 3; // list of requests IDs that are known by observer, but not requested by leader + bool enableMissingRequestRecovery = 4; } message RequestError { diff --git a/libs/go.mod b/libs/go.mod index 6957b42a0..72e1cdedd 100644 --- a/libs/go.mod +++ b/libs/go.mod @@ -8,7 +8,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-plugin v1.8.0 github.com/shopspring/decimal v1.4.0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260917115705-1d3a14a9b049 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260925190414-e1b87ad2d7b1 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 github.com/smartcontractkit/libocr v0.0.0-20260810200708-618b5bf7f342 github.com/stretchr/testify v1.12.1 diff --git a/libs/go.sum b/libs/go.sum index 8fbf69ac9..b832519a7 100644 --- a/libs/go.sum +++ b/libs/go.sum @@ -208,8 +208,8 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/smartcontractkit/chain-selectors v1.0.104 h1:/n9pPGM5W/+r1eHoWZv4VwX9LNS1af4+ICyhM8zKRNM= github.com/smartcontractkit/chain-selectors v1.0.104/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260917115705-1d3a14a9b049 h1:WebE9HTubitMC5xq+xCAArEI+oBLekVOoxYnCkB8whI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260917115705-1d3a14a9b049/go.mod h1:3CaCxSyZmgWl1PhxneFSZqOfN7DF2VnHML0bTfiDNIo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260925190414-e1b87ad2d7b1 h1:1rGIv4vUt+Et7k5waRbkODOQsMuEakQ8TA9PiMBaPss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260925190414-e1b87ad2d7b1/go.mod h1:UkZJ6uUU9Hlonfb+k62Pg4yRuvOsoShoOd4tPEB/qpE= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 h1:WEUMkKQPAgcNMRgES6CBWrRUiII+HKEWQjulKQBSuMA=