save val indexes and status on db

packages/brain/src/modules/cron/index.ts

@@ -49,7 +49,7 @@ export const getCrons = ({
       });
     }),
     reloadValidatorsCronTask: new CronJob(60 * 1000, () =>
-      reloadValidators(signerApi, signerUrl, validatorApi, brainDb)
+      reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb)
     )
   };
 };

packages/brain/src/modules/cron/reloadValidators/index.ts

@@ -1,4 +1,4 @@
-import { Web3SignerApi, ValidatorApi } from "../../apiClients/index.js";
+import { Web3SignerApi, ValidatorApi, BeaconchainApi } from "../../apiClients/index.js";
 import { BrainDataBase } from "../../db/index.js";
 import logger from "../../logger/index.js";
 import { deleteDbPubkeysNotInSigner } from "./deleteDbPubkeysNotInSigner.js";
@@ -7,6 +7,7 @@ import { deleteValidatorPubkeysNotInDB } from "./deleteValidatorPubkeysNotInDb.j
 import { logPrefix } from "./logPrefix.js";
 import { postValidatorPubkeysFromDb } from "./postValidatorPubkeysFromDb.js";
 import { postValidatorsFeeRecipientsFromDb } from "./postValidatorsFeeRecipientsFromDb.js";
+import { persistValidatorIndices } from "./persistValidatorIndices.js";
 
 /**
  * Reload db data based on truth sources: validator and signer APIs:
@@ -16,12 +17,14 @@ import { postValidatorsFeeRecipientsFromDb } from "./postValidatorsFeeRecipients
  * - DELETE from DB pubkeys that are not in signer API
  * - DELETE to validator API pubkeys that are in validator API and not in DB
  * - POST to validator API fee recipients that are in DB and not in validator API
+ * - FETCH and PERSIST validator indices from Beacon API for pubkeys without indices
  *
  */
 export async function reloadValidators(
   signerApi: Web3SignerApi,
   signerUrl: string,
   validatorApi: ValidatorApi,
+  beaconchainApi: BeaconchainApi,
   brainDb: BrainDataBase
 ): Promise<void> {
   try {
@@ -63,6 +66,12 @@ export async function reloadValidators(
       dbData: brainDb.getData()
     });
 
+    // 7. FETCH and PERSIST validator indices from Beacon API for all pubkeys in DB
+    await persistValidatorIndices({
+      beaconchainApi,
+      brainDb
+    });
+
     logger.debug(`${logPrefix}Finished reloading data`);
   } catch (e) {
     logger.error(`${logPrefix}Error reloading data`, e);

packages/brain/src/modules/cron/reloadValidators/persistValidatorIndices.ts

@@ -0,0 +1,106 @@
+import { BeaconchainApi } from "../../apiClients/index.js";
+import { BrainDataBase } from "../../db/index.js";
+import { ValidatorStatus } from "../../apiClients/beaconchain/types.js";
+import { shortenPubkey } from "@stakingbrain/common";
+import logger from "../../logger/index.js";
+import { logPrefix } from "./logPrefix.js";
+import { StakingBrainDb } from "../../db/types.js";
+
+interface ValidatorUpdate {
+  index: number;
+  status: ValidatorStatus;
+  feeRecipient: string;
+}
+
+interface UpdateResult {
+  validatorsToUpdate: Record<string, ValidatorUpdate>;
+  newIndicesCount: number;
+  statusChangesCount: number;
+}
+
+/**
+ * Fetches validator indices and statuses from the Beacon API for all validators
+ * in the database and persists any changes.
+ */
+export async function persistValidatorIndices({
+  beaconchainApi,
+  brainDb
+}: {
+  beaconchainApi: BeaconchainApi;
+  brainDb: BrainDataBase;
+}): Promise<void> {
+  try {
+    const dbData = brainDb.getData();
+    const allPubkeys = Object.keys(dbData);
+
+    if (allPubkeys.length === 0) {
+      logger.debug(`${logPrefix}No validators in database to fetch data for`);
+      return;
+    }
+
+    logger.debug(`${logPrefix}Fetching indices and statuses for ${allPubkeys.length} validators`);
+
+    const response = await beaconchainApi.postStateValidators({
+      stateId: "head",
+      body: { ids: allPubkeys, statuses: [] }
+    });
+
+    const { validatorsToUpdate, newIndicesCount, statusChangesCount } = processValidatorResponse(response.data, dbData);
+
+    const updateCount = Object.keys(validatorsToUpdate).length;
+    if (updateCount > 0) {
+      brainDb.updateValidators({ validators: validatorsToUpdate });
+      logger.debug(
+        `${logPrefix}Persisted ${updateCount} validator updates (${newIndicesCount} new indices, ${statusChangesCount} status changes)`
+      );
+    }
+  } catch (e) {
+    logger.error(`${logPrefix}Error persisting validator indices and statuses`, e);
+  }
+}
+
+/**
+ * Processes the beacon API response and identifies validators that need updating.
+ */
+function processValidatorResponse(
+  responseData: { index: string; status: ValidatorStatus; validator: { pubkey: string } }[],
+  dbData: StakingBrainDb
+): UpdateResult {
+  const validatorsToUpdate: Record<string, ValidatorUpdate> = {};
+  let newIndicesCount = 0;
+  let statusChangesCount = 0;
+
+  for (const validatorData of responseData) {
+    const pubkey = validatorData.validator.pubkey;
+    const dbEntry = dbData[pubkey];
+
+    if (!dbEntry) continue;
+
+    const newIndex = parseInt(validatorData.index);
+    const newStatus = validatorData.status;
+    const indexChanged = dbEntry.index !== newIndex;
+    const statusChanged = dbEntry.status !== newStatus;
+
+    if (!indexChanged && !statusChanged) continue;
+
+    validatorsToUpdate[pubkey] = {
+      index: newIndex,
+      status: newStatus,
+      feeRecipient: dbEntry.feeRecipient
+    };
+
+    if (dbEntry.index === undefined) {
+      newIndicesCount++;
+      logger.info(`${logPrefix}Validator ${shortenPubkey(pubkey)} assigned index ${newIndex} with status ${newStatus}`);
+    }
+
+    if (dbEntry.status !== undefined && statusChanged) {
+      statusChangesCount++;
+      logger.info(
+        `${logPrefix}Validator ${shortenPubkey(pubkey)} (index ${newIndex}) status changed: ${dbEntry.status} → ${newStatus}`
+      );
+    }
+  }
+
+  return { validatorsToUpdate, newIndicesCount, statusChangesCount };
+}

packages/brain/src/modules/db/index.ts

@@ -96,7 +96,7 @@ export class BrainDataBase extends LowSync<StakingBrainDb> {
   }
 
   /**
-   * Updates 1 or more validators in db. The fields available to update are feeRecipient and index
+   * Updates 1 or more validators in db. The fields available to update are feeRecipient, index, and status
    */
   public updateValidators({ validators }: { validators: StakingBrainDbUpdate }): void {
     try {
@@ -110,7 +110,14 @@ export class BrainDataBase extends LowSync<StakingBrainDb> {
             delete validators[pubkey];
           } else {
             this.data[pubkey].feeRecipient = validators[pubkey].feeRecipient;
-            this.data[pubkey].index = validators[pubkey].index;
+            // Optional fields. Only update if provided so we dont overwrite existing data with undefined
+            // Index cant change once defined by ethereum and status should change only a few times in a validator lifetime
+            if (validators[pubkey].index !== undefined) {
+              this.data[pubkey].index = validators[pubkey].index;
+            }
+            if (validators[pubkey].status !== undefined) {
+              this.data[pubkey].status = validators[pubkey].status;
+            }
           }
         }
 

packages/brain/src/modules/db/types.ts

@@ -1,4 +1,5 @@
 import { Tag } from "@stakingbrain/common";
+import { ValidatorStatus } from "../apiClients/beaconchain/types.js";
 
 /**
  * DbSlot represents the line in the database for a given public key:
@@ -25,6 +26,7 @@ export interface PubkeyDetails {
   feeRecipient: string;
   automaticImport: boolean;
   index?: number; // index of the validator. Only available if the validator is active.
+  status?: ValidatorStatus; // status of the validator from the Beacon API.
 }
 
 export const nonEditableFeeRecipientTags = ["rocketpool", "stader", "stakewise", "lido"] as const;

packages/brain/test/unit/modules/apiClients/cron.unit.test.ts

@@ -1,6 +1,6 @@
 import { expect } from "chai";
 import { before } from "mocha";
-import { ValidatorApi, Web3SignerApi } from "../../../../src/modules/apiClients/index.js";
+import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../../src/modules/apiClients/index.js";
 import { execSync } from "node:child_process";
 import { BrainDataBase } from "../../../../src/modules/db/index.js";
 import fs from "fs";
@@ -60,6 +60,7 @@ describe.skip("Cron: Prater", () => {
     describe(`Consensus client: ${consensusClient.name}`, () => {
       let validatorApi: ValidatorApi;
       let signerApi: Web3SignerApi;
+      let beaconchainApi: BeaconchainApi;
       let brainDb: BrainDataBase;
       let signerUrl: string;
 
@@ -97,6 +98,15 @@ describe.skip("Cron: Prater", () => {
         );
         signerUrl = `http://${signerIp}:9000`;
 
+        // Mock BeaconchainApi - postStateValidators returns empty array (validators not active yet)
+        beaconchainApi = {
+          postStateValidators: async () => ({
+            execution_optimistic: false,
+            finalized: true,
+            data: [] // No validators found on beacon chain yet
+          })
+        } as unknown as BeaconchainApi;
+
         if (fs.existsSync(testDbName)) fs.unlinkSync(testDbName);
         brainDb = new BrainDataBase(testDbName);
       });
@@ -132,7 +142,7 @@ describe.skip("Cron: Prater", () => {
         });
 
         //Check that fee recipient has changed in validator
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const validatorFeeRecipient = await validatorApi.getFeeRecipient(pubkeyToTest);
 
@@ -144,7 +154,7 @@ describe.skip("Cron: Prater", () => {
         addSampleValidatorsToDB(1);
         await addSampleKeystoresToSigner(2);
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const signerPubkeys = await signerApi.listRemoteKeys();
         const dbPubkeys = Object.keys(brainDb.getData());
@@ -159,7 +169,7 @@ describe.skip("Cron: Prater", () => {
         addSampleValidatorsToDB(2);
         await addSampleKeystoresToSigner(1);
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const signerPubkeys = await signerApi.listRemoteKeys();
         const dbPubkeys = Object.keys(brainDb.getData());
@@ -177,7 +187,7 @@ describe.skip("Cron: Prater", () => {
         brainDb.deleteValidators([pubkeys[0]]);
         await signerApi.deleteRemoteKeys({ pubkeys: [pubkeys[1]] });
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const signerPubkeys = await signerApi.listRemoteKeys();
         const dbPubkeys = Object.keys(brainDb.getData());
@@ -190,7 +200,7 @@ describe.skip("Cron: Prater", () => {
         addSampleValidatorsToDB(2);
         await addSampleKeystoresToSigner(2);
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const signerPubkeys = await signerApi.listRemoteKeys();
         const dbPubkeys = Object.keys(brainDb.getData());
@@ -208,7 +218,7 @@ describe.skip("Cron: Prater", () => {
 
         console.log("Added pubkeys to validator");
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         console.log("Validators reloaded");
 
@@ -225,7 +235,7 @@ describe.skip("Cron: Prater", () => {
 
         const pubkeysToTest = pubkeys.slice(0, 2);
 
-        await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
+        await reloadValidators(signerApi, signerUrl, validatorApi, beaconchainApi, brainDb);
 
         const validatorPubkeys = await validatorApi.getRemoteKeys();