From ac6a61df062b6078d9098015e0336a31d2a5aa24 Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Wed, 8 Jul 2026 22:52:58 +1000 Subject: [PATCH 1/2] fix(agent): skip a queued close when the rule now says allocate The reconcile loop executes allocation decisions computed from a rules snapshot up to several polling intervals old, so it closed allocations operators had just recreated. Closes decided by an opt-out rule now re-check the live rule and skip when it requests allocation. --- packages/indexer-agent/src/__tests__/agent.ts | 181 ++++++++++++++++++ packages/indexer-agent/src/agent.ts | 46 ++++- 2 files changed, 226 insertions(+), 1 deletion(-) diff --git a/packages/indexer-agent/src/__tests__/agent.ts b/packages/indexer-agent/src/__tests__/agent.ts index 7c2ac17f5..a91634b0f 100644 --- a/packages/indexer-agent/src/__tests__/agent.ts +++ b/packages/indexer-agent/src/__tests__/agent.ts @@ -434,6 +434,7 @@ describe('reconcileDeploymentAllocationAction', () => { 28, network, operator, + [], false, ) @@ -471,12 +472,192 @@ describe('reconcileDeploymentAllocationAction', () => { 28, network, operator, + [], false, ) expect(operator.createAllocation).not.toHaveBeenCalled() expect(network.networkMonitor.closedAllocations).not.toHaveBeenCalled() }) + + // The close decision embeds a rule snapshot up to several polling intervals + // old; the fresh-rules re-check must win when the two disagree. + describe('re-validates a close decision against the current rules', () => { + const closeDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.NEVER, + } as IndexingRuleAttributes, + false, + ActivationCriteria.NEVER, + 'eip155:42161', + ) + const currentRule = (basis: IndexingDecisionBasis) => + ({ + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: basis, + }) as IndexingRuleAttributes + + it('skips the close when the current rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + it('skips the close when the current rule says dips', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.DIPS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + it('closes when the current rule still opts out', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.NEVER)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + it('closes when no current rule exists for the deployment', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + it('skips the close when a global always rule governs the deployment', async () => { + const agent = createAgent() + const operator = createOperator() + const globalRule = { + identifier: INDEXING_RULE_GLOBAL, + identifierType: SubgraphIdentifierType.GROUP, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [globalRule], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + // A denied deployment yields toAllocate=false with UNSUPPORTED criteria + // even under an always rule; that close is current and must proceed. + it('closes an unsupported deployment even when the rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const unsupportedDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes, + false, + ActivationCriteria.UNSUPPORTED, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + unsupportedDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + // reconcileActions flips toAllocate to false for the network subgraph + // while its rule still says always; that deliberate override must win. + it('closes a config-overridden decision even when the rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const overriddenDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes, + false, + ActivationCriteria.ALWAYS, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + overriddenDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + }) }) describe('addIndexingPaymentsSubgraphToTarget function', () => { diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 1238d4e1a..fe295c0dd 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -9,12 +9,14 @@ import { } from '@graphprotocol/common-ts' import { ActionStatus, + ActivationCriteria, Allocation, AllocationManagementMode, allocationRewardsPool, AllocationStatus, indexerError, IndexerErrorCode, + INDEXING_RULE_GLOBAL, IndexingDecisionBasis, IndexerManagementClient, IndexingRuleAttributes, @@ -1036,6 +1038,7 @@ export class Agent { maxAllocationDuration: number, network: Network, operator: Operator, + currentIndexingRules: IndexingRuleAttributes[], forceAction: boolean = false, ): Promise { const logger = this.logger.child({ @@ -1053,13 +1056,44 @@ export class Agent { ) switch (deploymentAllocationDecision.toAllocate) { - case false: + case false: { + // A close decided purely by an opt-out rule (never/offchain) can be + // stale: the operator may have flipped the deployment back to allocate. + // Closes from other gates (unsupported, overrides) must still proceed. + const optOutDecision = + deploymentAllocationDecision.ruleMatch.activationCriteria === + ActivationCriteria.NEVER || + deploymentAllocationDecision.ruleMatch.activationCriteria === + ActivationCriteria.OFFCHAIN + if (optOutDecision) { + const currentRule = + currentIndexingRules.find( + rule => + rule.identifierType === SubgraphIdentifierType.DEPLOYMENT && + new SubgraphDeploymentID(rule.identifier).bytes32 === + deploymentAllocationDecision.deployment.bytes32, + ) ?? + currentIndexingRules.find( + rule => rule.identifier === INDEXING_RULE_GLOBAL, + ) + if ( + currentRule?.decisionBasis === IndexingDecisionBasis.ALWAYS || + currentRule?.decisionBasis === IndexingDecisionBasis.DIPS + ) { + logger.info( + `Skipping allocation close: the opt-out rule behind this decision has changed and the current rule requests allocation`, + { currentDecisionBasis: currentRule.decisionBasis }, + ) + return + } + } return await operator.closeEligibleAllocations( logger, deploymentAllocationDecision, activeDeploymentAllocations, forceAction, ) + } case true: { // If no active allocations and subgraph health passes safety check, create one const indexingStatuses = await this.graphNode.indexingStatus([ @@ -1221,6 +1255,15 @@ export class Agent { const activeAllocations: Allocation[] = await network.networkMonitor.allocations(AllocationStatus.ACTIVE) + // The decisions carry rules from a snapshot up to several polling + // intervals old; when this pass will close something, read the rules + // fresh too so opt-out closes re-validate against current intent. + const currentIndexingRules = allocationDecisions.some( + decision => !decision.toAllocate, + ) + ? await operator.indexingRules(true) + : [] + this.logger.trace(`Reconcile allocation actions`, { protocolNetwork: network.specification.networkIdentifier, epoch, @@ -1243,6 +1286,7 @@ export class Agent { maxAllocationDuration, network, operator, + currentIndexingRules, ), ) }, From 74ca3354d0d493fae180618a80fbed801104fd20 Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Thu, 9 Jul 2026 14:21:35 +1000 Subject: [PATCH 2/2] test(agent): pin the offchain flavour of the stale-close guard The guard treats never and offchain opt-out decisions identically, but only the never path had a direct test; the incident itself came through offchain. 1 test replays that flavour. --- packages/indexer-agent/src/__tests__/agent.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/indexer-agent/src/__tests__/agent.ts b/packages/indexer-agent/src/__tests__/agent.ts index a91634b0f..6e5b84681 100644 --- a/packages/indexer-agent/src/__tests__/agent.ts +++ b/packages/indexer-agent/src/__tests__/agent.ts @@ -537,6 +537,37 @@ describe('reconcileDeploymentAllocationAction', () => { expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() }) + // The incident's close was decided under an offchain rule (the management + // API's close stamp), so pin that flavour of the guard explicitly. + it('skips an offchain-decided close when the current rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const offchainDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.OFFCHAIN, + } as IndexingRuleAttributes, + false, + ActivationCriteria.OFFCHAIN, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + offchainDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + it('closes when the current rule still opts out', async () => { const agent = createAgent() const operator = createOperator()