diff --git a/packages/indexer-agent/src/__tests__/agent.ts b/packages/indexer-agent/src/__tests__/agent.ts index 7c2ac17f5..6e5b84681 100644 --- a/packages/indexer-agent/src/__tests__/agent.ts +++ b/packages/indexer-agent/src/__tests__/agent.ts @@ -434,6 +434,7 @@ describe('reconcileDeploymentAllocationAction', () => { 28, network, operator, + [], false, ) @@ -471,12 +472,223 @@ describe('reconcileDeploymentAllocationAction', () => { 28, network, operator, + [], false, ) expect(operator.createAllocation).not.toHaveBeenCalled() expect(network.networkMonitor.closedAllocations).not.toHaveBeenCalled() }) + + // The close decision embeds a rule snapshot up to several polling intervals + // old; the fresh-rules re-check must win when the two disagree. + describe('re-validates a close decision against the current rules', () => { + const closeDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.NEVER, + } as IndexingRuleAttributes, + false, + ActivationCriteria.NEVER, + 'eip155:42161', + ) + const currentRule = (basis: IndexingDecisionBasis) => + ({ + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: basis, + }) as IndexingRuleAttributes + + it('skips the close when the current rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + it('skips the close when the current rule says dips', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.DIPS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + // The incident's close was decided under an offchain rule (the management + // API's close stamp), so pin that flavour of the guard explicitly. + it('skips an offchain-decided close when the current rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const offchainDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.OFFCHAIN, + } as IndexingRuleAttributes, + false, + ActivationCriteria.OFFCHAIN, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + offchainDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + it('closes when the current rule still opts out', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.NEVER)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + it('closes when no current rule exists for the deployment', async () => { + const agent = createAgent() + const operator = createOperator() + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + it('skips the close when a global always rule governs the deployment', async () => { + const agent = createAgent() + const operator = createOperator() + const globalRule = { + identifier: INDEXING_RULE_GLOBAL, + identifierType: SubgraphIdentifierType.GROUP, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes + + await agent.reconcileDeploymentAllocationAction( + closeDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [globalRule], + false, + ) + + expect(operator.closeEligibleAllocations).not.toHaveBeenCalled() + }) + + // A denied deployment yields toAllocate=false with UNSUPPORTED criteria + // even under an always rule; that close is current and must proceed. + it('closes an unsupported deployment even when the rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const unsupportedDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes, + false, + ActivationCriteria.UNSUPPORTED, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + unsupportedDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + + // reconcileActions flips toAllocate to false for the network subgraph + // while its rule still says always; that deliberate override must win. + it('closes a config-overridden decision even when the rule says always', async () => { + const agent = createAgent() + const operator = createOperator() + const overriddenDecision = new AllocationDecision( + deployment, + { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + } as IndexingRuleAttributes, + false, + ActivationCriteria.ALWAYS, + 'eip155:42161', + ) + + await agent.reconcileDeploymentAllocationAction( + overriddenDecision, + activeAllocations, + 10, + 28, + createNetwork(), + operator, + [currentRule(IndexingDecisionBasis.ALWAYS)], + false, + ) + + expect(operator.closeEligibleAllocations).toHaveBeenCalled() + }) + }) }) describe('addIndexingPaymentsSubgraphToTarget function', () => { diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 1238d4e1a..fe295c0dd 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -9,12 +9,14 @@ import { } from '@graphprotocol/common-ts' import { ActionStatus, + ActivationCriteria, Allocation, AllocationManagementMode, allocationRewardsPool, AllocationStatus, indexerError, IndexerErrorCode, + INDEXING_RULE_GLOBAL, IndexingDecisionBasis, IndexerManagementClient, IndexingRuleAttributes, @@ -1036,6 +1038,7 @@ export class Agent { maxAllocationDuration: number, network: Network, operator: Operator, + currentIndexingRules: IndexingRuleAttributes[], forceAction: boolean = false, ): Promise { const logger = this.logger.child({ @@ -1053,13 +1056,44 @@ export class Agent { ) switch (deploymentAllocationDecision.toAllocate) { - case false: + case false: { + // A close decided purely by an opt-out rule (never/offchain) can be + // stale: the operator may have flipped the deployment back to allocate. + // Closes from other gates (unsupported, overrides) must still proceed. + const optOutDecision = + deploymentAllocationDecision.ruleMatch.activationCriteria === + ActivationCriteria.NEVER || + deploymentAllocationDecision.ruleMatch.activationCriteria === + ActivationCriteria.OFFCHAIN + if (optOutDecision) { + const currentRule = + currentIndexingRules.find( + rule => + rule.identifierType === SubgraphIdentifierType.DEPLOYMENT && + new SubgraphDeploymentID(rule.identifier).bytes32 === + deploymentAllocationDecision.deployment.bytes32, + ) ?? + currentIndexingRules.find( + rule => rule.identifier === INDEXING_RULE_GLOBAL, + ) + if ( + currentRule?.decisionBasis === IndexingDecisionBasis.ALWAYS || + currentRule?.decisionBasis === IndexingDecisionBasis.DIPS + ) { + logger.info( + `Skipping allocation close: the opt-out rule behind this decision has changed and the current rule requests allocation`, + { currentDecisionBasis: currentRule.decisionBasis }, + ) + return + } + } return await operator.closeEligibleAllocations( logger, deploymentAllocationDecision, activeDeploymentAllocations, forceAction, ) + } case true: { // If no active allocations and subgraph health passes safety check, create one const indexingStatuses = await this.graphNode.indexingStatus([ @@ -1221,6 +1255,15 @@ export class Agent { const activeAllocations: Allocation[] = await network.networkMonitor.allocations(AllocationStatus.ACTIVE) + // The decisions carry rules from a snapshot up to several polling + // intervals old; when this pass will close something, read the rules + // fresh too so opt-out closes re-validate against current intent. + const currentIndexingRules = allocationDecisions.some( + decision => !decision.toAllocate, + ) + ? await operator.indexingRules(true) + : [] + this.logger.trace(`Reconcile allocation actions`, { protocolNetwork: network.specification.networkIdentifier, epoch, @@ -1243,6 +1286,7 @@ export class Agent { maxAllocationDuration, network, operator, + currentIndexingRules, ), ) },