Merge pull request #2714 from w3f/will-v3.1.5

v3.1.5

Author: wpank

apps/1kv-backend/templates/kusama-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.4
+    targetRevision: v3.1.5
     plugin:
       env:
         - name: HELM_VALUES

apps/1kv-backend/templates/polkadot-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.4
+    targetRevision: v3.1.5
     plugin:
       env:
         - name: HELM_VALUES

charts/otv-backend/Chart.yaml

@@ -1,5 +1,5 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v3.1.4
-appVersion: v3.1.4
+version: v3.1.5
+appVersion: v3.1.5
 apiVersion: v2

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1kv/common",
-  "version": "3.1.4",
+  "version": "3.1.5",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",

packages/common/src/constants.ts

@@ -81,18 +81,18 @@ export const defaultWsOptions = {
   debug: true,
 };
 
-// 200 hours in milliseconds
-export const STALE_TELEMETRY_THRESHOLD = 720000000;
+// 2 days in milliseconds
+export const STALE_TELEMETRY_THRESHOLD = 172800000;
 
 export const DEFAULT_TELEMETRY_ENDPONT =
   "wss://telemetry.w3f.community/submit/";
 
 // List of log labels that are omitted from logging
 export const defaultExcludeLabels = [
-  "Telemetry",
-  "Location",
+  // "Telemetry",
+  // "Location",
   // "ValidatorPrefJob",
-  // "Block",
+  "Block",
   "Gateway",
 ];
 

packages/common/src/constraints/ScoreCandidates.ts

@@ -11,6 +11,7 @@ import {
 } from "../db";
 import { constraintsLabel, OTV } from "./constraints";
 import { percentage, timeRemaining } from "../utils/util";
+import { NoLocation } from "../types";
 
 export const scoreCandidate = async (
   constraints: OTV,
@@ -94,7 +95,9 @@ export const scoreCandidate = async (
       (1 - scaledFaults) * constraints.WEIGHT_CONFIG.FAULTS_WEIGHT;
 
     const latestCandidateLocation = await queries.getCandidateLocation(
+      candidate.slotId,
       candidate.name,
+      candidate.stash,
     );
 
     const provider = latestCandidateLocation?.provider || "No Provider";
@@ -116,7 +119,9 @@ export const scoreCandidate = async (
       scaledDefined(candidateLocation, locationValues, 0, 1) || 0;
     const locationScore = bannedProvider
       ? 0
-      : latestCandidateLocation?.city == "No Location"
+      : latestCandidateLocation?.city == NoLocation.NoLocation ||
+          !latestCandidateLocation ||
+          !latestCandidateLocation?.city
         ? 0.25 * constraints.WEIGHT_CONFIG.LOCATION_WEIGHT
         : (1 - scaledLocation) * constraints.WEIGHT_CONFIG.LOCATION_WEIGHT || 0;
 
@@ -135,7 +140,9 @@ export const scoreCandidate = async (
       scaledDefined(candidateRegion, regionValues, 0, 1) || 0;
     const regionScore = bannedProvider
       ? 0
-      : latestCandidateLocation?.region == "No Location"
+      : latestCandidateLocation?.region == NoLocation.NoLocation ||
+          !latestCandidateLocation ||
+          !latestCandidateLocation?.region
         ? 0.25 * constraints.WEIGHT_CONFIG.REGION_WEIGHT
         : (1 - scaledRegion) * constraints.WEIGHT_CONFIG.REGION_WEIGHT || 0;
 
@@ -154,7 +161,9 @@ export const scoreCandidate = async (
       scaledDefined(candidateCountry, countryValues, 0, 1) || 0;
     const countryScore = bannedProvider
       ? 0
-      : latestCandidateLocation?.country == "No Location"
+      : latestCandidateLocation?.country == NoLocation.NoLocation ||
+          !latestCandidateLocation ||
+          !latestCandidateLocation?.country
         ? 0.25 * constraints.WEIGHT_CONFIG.COUNTRY_WEIGHT
         : (1 - scaledCountry) * constraints.WEIGHT_CONFIG.COUNTRY_WEIGHT || 0;
 
@@ -173,7 +182,9 @@ export const scoreCandidate = async (
       scaledDefined(candidateProvider, providerValues, 0, 1) || 0;
     const providerScore = bannedProvider
       ? 0
-      : latestCandidateLocation?.provider == "No Location"
+      : latestCandidateLocation?.provider == "No Location" ||
+          !latestCandidateLocation ||
+          !latestCandidateLocation?.provider
         ? 0.25 * constraints.WEIGHT_CONFIG.PROVIDER_WEIGHT
         : (1 - scaledProvider) * constraints.WEIGHT_CONFIG.PROVIDER_WEIGHT || 0;
 

packages/common/src/constraints/ValidityChecks.ts

@@ -325,7 +325,11 @@ export const checkProvider = async (
   candidate: Candidate,
 ): Promise<boolean> => {
   try {
-    const location = await queries.getCandidateLocation(candidate.name);
+    const location = await queries.getCandidateLocation(
+      candidate.slotId,
+      candidate.name,
+      candidate.stash,
+    );
     if (location && location.provider) {
       const bannedProviders = config.telemetry?.blacklistedProviders;
       if (bannedProviders?.includes(location.provider)) {

packages/common/src/constraints/Values.ts

@@ -7,6 +7,7 @@ import {
   getLatestNominatorStake,
 } from "../db";
 import { constraintsLabel } from "./constraints";
+import { NoLocation } from "../types";
 
 export const getBondedValues = (
   validCandidates: Candidate[],
@@ -116,8 +117,12 @@ export const getLocationValues = async (
   const locationMap = new Map<string, number>();
   const locationArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(candidate.name);
-    const location = candidateLocation?.city || "No Location";
+    const candidateLocation = await getCandidateLocation(
+      candidate.slotId,
+      candidate.name,
+      candidate.stash,
+    );
+    const location = candidateLocation?.city || NoLocation.NoLocation;
 
     const locationCount = locationMap.get(location);
     if (!locationCount) {
@@ -150,11 +155,15 @@ export const getRegionValues = async (
   const regionMap = new Map<string, number>();
   const regionArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(candidate.name);
+    const candidateLocation = await getCandidateLocation(
+      candidate.slotId,
+      candidate.name,
+      candidate.stash,
+    );
     const region =
       candidateLocation && candidateLocation.region
         ? candidateLocation.region
-        : "No Location";
+        : NoLocation.NoLocation;
 
     const regionCount = regionMap.get(region);
     if (!regionCount) {
@@ -187,11 +196,15 @@ export const getCountryValues = async (
   const countryMap = new Map<string, number>();
   const countryArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(candidate.name);
+    const candidateLocation = await getCandidateLocation(
+      candidate.slotId,
+      candidate.name,
+      candidate.stash,
+    );
     const country =
       candidateLocation && candidateLocation.country
         ? candidateLocation.country
-        : "No Location";
+        : NoLocation.NoLocation;
 
     const countryCount = countryMap.get(country);
     if (!countryCount) {
@@ -222,11 +235,15 @@ export const getProviderValues = async (
   const providerMap = new Map<string, number>();
   const providerArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(candidate.name);
+    const candidateLocation = await getCandidateLocation(
+      candidate.slotId,
+      candidate.name,
+      candidate.stash,
+    );
     const provider =
       candidateLocation && candidateLocation.provider
         ? candidateLocation.provider
-        : "No Location";
+        : NoLocation.NoProvider;
 
     const providerCount = providerMap.get(provider);
     if (!providerCount) {

packages/common/src/db/models.ts

@@ -189,8 +189,13 @@ export const ValidatorSetModel = mongoose.model(
 );
 
 export interface Location {
+  // The SlotId of the candidate
+  slotId: number;
+  // The Telemetry name of the candidate
   name: string;
+  // The stash address of the candidate
   address: string;
+  // The IP address of the candidate
   addr: string;
   port: number;
   city: string;
@@ -208,6 +213,7 @@ export interface Location {
 }
 
 export const LocationSchema = new Schema({
+  slotId: { type: Number, index: true }, // The Candidate SlotId
   name: { type: String, index: true }, // The Telemetry name of the node
   address: { type: String, index: true }, // The Validator Address
   addr: { type: String, index: true },

packages/common/src/db/queries/Candidate.ts

@@ -887,22 +887,16 @@ export const invalidCandidates = async (): Promise<any[]> => {
   return CandidateModel.find({ valid: false }).lean<Candidate[]>();
 };
 
-/**
- * Gets a candidate by its stash address.
- * @param stashOrName The DOT / KSM address or the name of the validator.
- */
 export const getCandidate = async (
-  stashOrName: string,
+  slotId: number,
+  stash?: string,
+  address?: string,
 ): Promise<Candidate | null> => {
-  let data = await CandidateModel.findOne({
-    stash: stashOrName,
-  }).lean<Candidate>();
+  const query: any = { slotId };
+  if (stash) query.stash = stash;
+  if (address) query.address = address;
 
-  if (!data) {
-    data = await CandidateModel.findOne({
-      name: stashOrName,
-    }).lean<Candidate>();
-  }
+  const data = await CandidateModel.findOne(query).lean<Candidate>();
 
   return data;
 };
@@ -919,6 +913,12 @@ export const getCandidateBySlotId = async (
   return CandidateModel.findOne({ slotId: id }).lean<Candidate>();
 };
 
+export const getCandidateByStash = async (
+  stash: string,
+): Promise<Candidate | null> => {
+  return CandidateModel.findOne({ stash: stash }).lean<Candidate>();
+};
+
 export const setInclusion = async (
   address: string,
   inclusion: number,
@@ -1709,7 +1709,7 @@ export const getUniqueStashSet = async (): Promise<any> => {
 };
 
 export const isKYC = async (stash: string): Promise<boolean | null> => {
-  const candidate = await getCandidate(stash);
+  const candidate = await getCandidateByStash(stash);
   if (candidate) {
     return candidate.kyc;
   }