refactor/provider-endpoint-attributes

api/package.json

@@ -54,7 +54,7 @@
     "@types/node": "^16.6.0",
     "@types/node-fetch": "^2.6.2",
     "@types/pg": "^8.6.5",
-    "@types/semver": "^7.5.1",
+    "@types/semver": "^7.5.2",
     "@types/uuid": "^8.3.1",
     "alias-hq": "^5.1.6",
     "nodemon": "^2.0.7",

api/src/db/blocksProvider.ts

@@ -1,6 +1,6 @@
 import { AkashBlock as Block, AkashMessage as Message } from "@shared/dbSchemas/akash";
 import { Transaction, Validator } from "@shared/dbSchemas/base";
-import { averageBlockTime } from "@src/shared/constants";
+import { averageBlockTime } from "@src/utils/constants";
 import { add } from "date-fns";
 
 export async function getBlocks(limit: number) {

api/src/db/dbConnection.ts

@@ -1,5 +1,5 @@
 import { defaults as pgDefaults } from "pg";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { Transaction as DbTransaction } from "sequelize";
 import { Sequelize } from "sequelize-typescript";
 import { chainModels, getChainModels, userModels } from "@shared/dbSchemas";

api/src/db/deploymentProvider.ts

@@ -1,6 +1,6 @@
 import * as v1 from "@src/proto/akash/v1beta1";
 import * as v2 from "@src/proto/akash/v1beta2";
-import { decodeMsg } from "@src/shared/utils/protobuf";
+import { decodeMsg } from "@src/utils/protobuf";
 import { Transaction } from "@shared/dbSchemas/base";
 import { Deployment } from "@shared/dbSchemas/akash";
 import { Op } from "sequelize";

api/src/db/networkRevenueProvider.ts

@@ -2,8 +2,8 @@ import { Op } from "sequelize";
 import { Day } from "@shared/dbSchemas/base";
 import { AkashBlock as Block } from "@shared/dbSchemas/akash";
 import { add } from "date-fns";
-import { getTodayUTC } from "@src/shared/utils/date";
-import { round, uaktToAKT, udenomToDenom } from "@src/shared/utils/math";
+import { getTodayUTC } from "@src/utils/date";
+import { round, uaktToAKT, udenomToDenom } from "@src/utils/math";
 
 export const getWeb3IndexRevenue = async (debug?: boolean) => {
   const dailyNetworkRevenues = await getDailyRevenue();

api/src/db/templateProvider.ts

@@ -1,7 +1,7 @@
 import { Template, TemplateFavorite, UserSetting } from "@shared/dbSchemas/user";
 import * as uuid from "uuid";
 import { Op } from "sequelize";
-import { toUTC } from "@src/shared/utils/date";
+import { toUTC } from "@src/utils/date";
 
 export async function getTemplateById(id: string, userId: string = "") {
   const template = await Template.findOne({

api/src/db/transactionsProvider.ts

@@ -1,6 +1,6 @@
 import { Transaction, AddressReference } from "@shared/dbSchemas/base";
 import { AkashBlock as Block, AkashMessage as Message } from "@shared/dbSchemas/akash";
-import { msgToJSON } from "@src/shared/utils/protobuf";
+import { msgToJSON } from "@src/utils/protobuf";
 import { QueryTypes } from "sequelize";
 import { chainDb } from "./dbConnection";
 

api/src/index.ts

@@ -1,15 +1,14 @@
 import express from "express";
 import cors from "cors";
 import packageJson from "../package.json";
-import { isProd } from "./shared/constants";
+import { isProd } from "./utils/constants";
 import * as Sentry from "@sentry/node";
-import * as marketDataProvider from "./providers/marketDataProvider";
 import { Scheduler } from "./scheduler";
 import { apiRouter } from "./routers/apiRouter";
 import { userRouter } from "./routers/userRouter";
 import { web3IndexRouter } from "./routers/web3indexRouter";
-import { bytesToHumanReadableSize } from "./shared/utils/files";
-import { env } from "./shared/utils/env";
+import { bytesToHumanReadableSize } from "./utils/files";
+import { env } from "./utils/env";
 import { chainDb, syncUserSchema, userDb } from "./db/dbConnection";
 import { dashboardRouter } from "./routers/dashboardRouter";
 

api/src/middlewares/privateMiddleware.ts

@@ -1,4 +1,4 @@
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 
 export function privateMiddleware(req, res, next) {
   if (!env.SecretToken) {

api/src/middlewares/userMiddleware.ts

@@ -1,4 +1,4 @@
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { expressjwt as jwt, GetVerificationKey, Request as JWTRequest } from "express-jwt";
 import { expressJwtSecret } from "jwks-rsa";
 

api/src/providers/apiNodeProvider.ts

@@ -1,15 +1,14 @@
 import fetch from "node-fetch";
 import { getDeploymentRelatedMessages } from "../db/deploymentProvider";
-import { averageBlockCountInAMonth } from "@src/shared/constants";
-import { getMarketData } from "./marketDataProvider";
-import { coinToAsset } from "@src/shared/utils/coin";
+import { averageBlockCountInAMonth } from "@src/utils/constants";
+import { coinToAsset } from "@src/utils/coin";
 import { getTransactionByAddress } from "@src/db/transactionsProvider";
 import axios from "axios";
 import { Validator } from "@shared/dbSchemas/base";
 import { Op } from "sequelize";
 import { Provider, ProviderAttribute } from "@shared/dbSchemas/akash";
 import { cacheKeys, cacheResponse } from "@src/caching/helpers";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { RestDeploymentInfoResponse, RestLeaseListResponse } from "@src/types/rest";
 
 const defaultNodeUrlMapping = {
@@ -349,8 +348,6 @@ export async function getDeployment(owner: string, dseq: string) {
     },
     include: [{ model: ProviderAttribute }]
   });
-
-  const marketData = await cacheResponse(60 * 5, cacheKeys.getMarketData, getMarketData);
   const deploymentDenom = deploymentData.escrow_account.balance.denom;
 
   const leases = leasesData.leases.map((x) => {

api/src/providers/githubProvider.ts

@@ -1,5 +1,8 @@
 import { Octokit } from "@octokit/rest";
-import { env } from "@src/shared/utils/env";
+import { cacheKeys, cacheResponse } from "@src/caching/helpers";
+import { ProviderAttributesSchema } from "@src/types/provider";
+import { env } from "@src/utils/env";
+import axios from "axios";
 
 export function getOctokit() {
   const githubPAT = env.AkashlyticsGithubPAT;
@@ -14,3 +17,23 @@ export function getOctokit() {
     baseUrl: "https://api.github.com"
   });
 }
+
+export const getProviderAttributesSchema = async (): Promise<ProviderAttributesSchema> => {
+  // Fetching provider attributes schema
+  const response = await cacheResponse(
+    30,
+    cacheKeys.getProviderAttributesSchema,
+    async () => await axios.get("https://raw.githubusercontent.com/akash-network/cloudmos/main/config/provider-attributes.json")
+  );
+
+  return response.data;
+};
+
+export async function getAuditors() {
+  const response = await cacheResponse(60 * 5, cacheKeys.getAuditors, async () => {
+    const res = await axios.get("https://raw.githubusercontent.com/akash-network/cloudmos/main/config/auditors.json");
+    return res.data;
+  });
+
+  return response;
+}

api/src/providers/providerStatusProvider.ts

@@ -1,9 +1,11 @@
 import { Provider, ProviderAttribute, ProviderAttributeSignature } from "@shared/dbSchemas/akash";
 import { ProviderSnapshot } from "@shared/dbSchemas/akash/providerSnapshot";
-import { toUTC } from "@src/shared/utils/date";
+import { toUTC } from "@src/utils/date";
 import { add } from "date-fns";
 import { Op } from "sequelize";
 import semver from "semver";
+import { mapProviderToList } from "@src/utils/map/provider";
+import { getAuditors, getProviderAttributesSchema } from "./githubProvider";
 
 export async function getNetworkCapacity() {
   const providers = await Provider.findAll({
@@ -12,21 +14,22 @@ export async function getNetworkCapacity() {
       deletedHeight: null
     }
   });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
 
   const stats = {
-    activeProviderCount: providers.length,
-    activeCPU: providers.map((x) => x.activeCPU).reduce((a, b) => a + b, 0),
-    activeGPU: providers.map((x) => x.activeGPU).reduce((a, b) => a + b, 0),
-    activeMemory: providers.map((x) => x.activeMemory).reduce((a, b) => a + b, 0),
-    activeStorage: providers.map((x) => x.activeStorage).reduce((a, b) => a + b, 0),
-    pendingCPU: providers.map((x) => x.pendingCPU).reduce((a, b) => a + b, 0),
-    pendingGPU: providers.map((x) => x.pendingGPU).reduce((a, b) => a + b, 0),
-    pendingMemory: providers.map((x) => x.pendingMemory).reduce((a, b) => a + b, 0),
-    pendingStorage: providers.map((x) => x.pendingStorage).reduce((a, b) => a + b, 0),
-    availableCPU: providers.map((x) => x.availableCPU).reduce((a, b) => a + b, 0),
-    availableGPU: providers.map((x) => x.availableGPU).reduce((a, b) => a + b, 0),
-    availableMemory: providers.map((x) => x.availableMemory).reduce((a, b) => a + b, 0),
-    availableStorage: providers.map((x) => x.availableStorage).reduce((a, b) => a + b, 0)
+    activeProviderCount: filteredProviders.length,
+    activeCPU: filteredProviders.map((x) => x.activeCPU).reduce((a, b) => a + b, 0),
+    activeGPU: filteredProviders.map((x) => x.activeGPU).reduce((a, b) => a + b, 0),
+    activeMemory: filteredProviders.map((x) => x.activeMemory).reduce((a, b) => a + b, 0),
+    activeStorage: filteredProviders.map((x) => x.activeStorage).reduce((a, b) => a + b, 0),
+    pendingCPU: filteredProviders.map((x) => x.pendingCPU).reduce((a, b) => a + b, 0),
+    pendingGPU: filteredProviders.map((x) => x.pendingGPU).reduce((a, b) => a + b, 0),
+    pendingMemory: filteredProviders.map((x) => x.pendingMemory).reduce((a, b) => a + b, 0),
+    pendingStorage: filteredProviders.map((x) => x.pendingStorage).reduce((a, b) => a + b, 0),
+    availableCPU: filteredProviders.map((x) => x.availableCPU).reduce((a, b) => a + b, 0),
+    availableGPU: filteredProviders.map((x) => x.availableGPU).reduce((a, b) => a + b, 0),
+    availableMemory: filteredProviders.map((x) => x.availableMemory).reduce((a, b) => a + b, 0),
+    availableStorage: filteredProviders.map((x) => x.availableStorage).reduce((a, b) => a + b, 0)
   };
 
   return {
@@ -66,11 +69,11 @@ export async function getProviders() {
       }
     ]
   });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
 
-  return providers.map((x) => {
+  return filteredProviders.map((x) => {
     const isValidVersion = x.cosmosSdkVersion ? semver.gte(x.cosmosSdkVersion, "v0.45.9") : false;
-    const name = new URL(x.hostUri).hostname;
-    const uptime7d = x.providerSnapshots.some((ps) => ps.isOnline) ? x.providerSnapshots.filter((ps) => ps.isOnline).length / x.providerSnapshots.length : 0;
+    const name = x.isOnline ? new URL(x.hostUri).hostname : null;
 
     return {
       owner: x.owner,
@@ -113,15 +116,39 @@ export async function getProviders() {
         memory: isValidVersion ? x.availableMemory : 0,
         storage: isValidVersion ? x.availableStorage : 0
       },
-      uptime7d: uptime7d,
+      uptime7d: x.uptime7d,
       uptime: x.providerSnapshots
         .filter((ps) => ps.checkDate > add(nowUtc, { days: -1 }))
         .map((ps) => ({
           id: ps.id,
           isOnline: ps.isOnline,
           checkDate: ps.checkDate
         })),
-      isValidVersion
+      isValidVersion,
+      isOnline: x.isOnline
     };
   });
 }
+
+export const getProviderList = async () => {
+  const providers = await Provider.findAll({
+    where: {
+      deletedHeight: null
+    },
+    include: [
+      {
+        model: ProviderAttribute
+      },
+      {
+        model: ProviderAttributeSignature
+      }
+    ]
+  });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
+  const providerAttributeSchemaQuery = getProviderAttributesSchema();
+  const auditorsQuery = getAuditors();
+
+  const [auditors, providerAttributeSchema] = await Promise.all([auditorsQuery, providerAttributeSchemaQuery]);
+
+  return filteredProviders.map((x) => mapProviderToList(x, providerAttributeSchema, auditors));
+};

api/src/providers/stripeProvider.ts

@@ -1,6 +1,6 @@
 import { UserSetting } from "@shared/dbSchemas/user";
 import { PlanCode } from "@shared/plans";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 
 import Stripe from "stripe";
 

api/src/providers/templateReposProvider.ts

@@ -2,11 +2,11 @@ import fetch from "node-fetch";
 import { markdownToTxt } from "markdown-to-txt";
 import path from "path";
 import { getOctokit } from "./githubProvider";
-import { isUrlAbsolute } from "@src/shared/utils/urls";
+import { isUrlAbsolute } from "@src/utils/urls";
 import * as fs from "fs";
 import { Octokit } from "@octokit/rest";
 import { getLogoFromPath } from "./templateReposLogos";
-import { dataFolderPath } from "@src/shared/constants";
+import { dataFolderPath } from "@src/utils/constants";
 import { GithubChainRegistryAssetListResponse } from "@src/types";
 import { GithubDirectoryItem } from "@src/types/github";
 

api/src/routers/apiRouter.ts

@@ -12,17 +12,17 @@ import {
   getValidator,
   getValidators
 } from "@src/providers/apiNodeProvider";
-import { getNetworkCapacity, getProviders } from "@src/providers/providerStatusProvider";
+import { getNetworkCapacity, getProviderList, getProviders } from "@src/providers/providerStatusProvider";
 import { getDashboardData, getGraphData, getProviderActiveLeasesGraphData, getProviderGraphData } from "@src/db/statsProvider";
-import { round } from "@src/shared/utils/math";
-import { isValidBech32Address } from "@src/shared/utils/addresses";
-import { getAkashPricing, getAWSPricing, getAzurePricing, getGCPPricing } from "@src/shared/utils/pricing";
+import { round } from "@src/utils/math";
+import { isValidBech32Address } from "@src/utils/addresses";
+import { getAkashPricing, getAWSPricing, getAzurePricing, getGCPPricing } from "@src/utils/pricing";
 import asyncHandler from "express-async-handler";
 import { ProviderStatsKey } from "@src/types/graph";
-import { getProviderAttributesSchema } from "@src/providers/providerAttributesProvider";
 import { cacheKeys, cacheResponse } from "@src/caching/helpers";
 import axios from "axios";
 import { getMarketData } from "@src/providers/marketDataProvider";
+import { getAuditors, getProviderAttributesSchema } from "@src/providers/githubProvider";
 
 export const apiRouter = express.Router();
 
@@ -233,6 +233,14 @@ apiRouter.get(
   })
 );
 
+apiRouter.get(
+  "/providerList",
+  asyncHandler(async (req, res) => {
+    const providers = await getProviderList();
+    res.send(providers);
+  })
+);
+
 apiRouter.get("/marketData", async (req, res) => {
   const response = await cacheResponse(60 * 5, cacheKeys.getMarketData, getMarketData);
   res.send(response);
@@ -371,10 +379,7 @@ apiRouter.get(
 apiRouter.get(
   "/getAuditors",
   asyncHandler(async (req, res) => {
-    const response = await cacheResponse(60 * 5, cacheKeys.getAuditors, async () => {
-      const res = await axios.get("https://raw.githubusercontent.com/akash-network/cloudmos/main/config/auditors.json");
-      return res.data;
-    });
+    const response = await getAuditors();
     res.send(response);
   })
 );

api/src/routers/userRouter.ts

@@ -10,7 +10,7 @@ import {
   checkUsernameAvailable,
   subscribeToNewsletter
 } from "@src/providers/userDataProvider";
-import { isValidBech32Address } from "@src/shared/utils/addresses";
+import { isValidBech32Address } from "@src/utils/addresses";
 import * as uuid from "uuid";
 import {
   deleteTemplate,

api/src/scheduler.ts

@@ -1,5 +1,5 @@
 import humanInterval from "human-interval";
-import { getPrettyTime } from "./shared/utils/date";
+import { getPrettyTime } from "./utils/date";
 import fetch from "node-fetch";
 
 class TaskDef {