initialize

.gitignore

@@ -0,0 +1,7 @@
+# typescripts
+/src/config.ts
+
+# resources
+/node_modules
+/shellScripts/gitConfig.sh
+package-lock.json

README.md

@@ -0,0 +1 @@
+# validator-identity-crawler

nodemon.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src", ".env"],
+  "ext": "js,ts,json",
+  "exec": "ts-node -r ./src/server.ts",
+  "ignore": ["src/scheduler.ts"]
+}

package.json

@@ -0,0 +1,22 @@
+{
+  "name": "validator-identity-crawler",
+  "version": "0.0.1",
+  "description": "The Official FirmaChain Javascript SDK written in Typescript",
+  "author": "FirmaChain <[email protected]>",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "start": "ts-node src/crawler.ts",
+    "start:dev": "ts-node src/crawler.ts --watch",
+    "start:debug": "ts-node src/crawler.ts --debug --watch"
+  },
+  "keywords": [],
+  "devDependencies": {
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.4"
+  },
+  "dependencies": {
+    "@firmachain/firma-js": "^0.2.47",
+    "child_process": "^1.0.2"
+  }
+}

public/identity.json

@@ -0,0 +1 @@
+{"profileInfos":[],"lastUpdatedTime":0}

shellScripts/gitAdd.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+git add public/identity.json

shellScripts/gitCommit.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+git commit -m "update: identity.json file has been updated"

shellScripts/gitConfig.sample.sh

@@ -0,0 +1,2 @@
+git config --global user.name "your name"
+git config --global user.email "your email"

shellScripts/gitPull.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+git pull

shellScripts/gitPush.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+git push -u origin master

src/config.sample.ts

@@ -0,0 +1,12 @@
+import { FirmaConfig } from "@firmachain/firma-js";
+
+export const FIRMACHAIN_CONFIG = FirmaConfig.MainNetConfig;
+export const KEYBASE_URL = "https://keybase.io/_/api/1.0/user";
+export const KEYBASE_PATH = "/lookup.json?fields=pictures&key_suffix=";
+export const PUBLIC_PATH = `${__dirname}/../public/`;
+export const IDENTITY_FILE_NAME = "identity.json";
+export const GIT_CONFIG_FILE_NAME = "bash shellScripts/gitConfig.sh";
+export const GIT_PULL_FILE_NAME = "bash shellScripts/gitPull.sh";
+export const GIT_ADD_FILE_NAME = "bash shellScripts/gitAdd.sh";
+export const GIT_COMMIT_FILE_NAME = "bash shellScripts/gitCommit.sh";
+export const GIT_PUSH_FILE_NAME = "bash shellScripts/gitPush.sh";

src/crawler.ts

@@ -0,0 +1,75 @@
+import { getIdentityFile, writeIdentityFile } from "./utils/file";
+import { IdentityFileInfo } from "./interfaces/common";
+import {
+  gitPushProcess,
+  InitGitConfig
+} from "./utils/gitUtil";
+import {
+  compareIdentityInfo,
+  getImageUrlByIdentityInfos,
+  getValidatorList,
+  organizeIdentity
+} from "./utils/crawlerUtil";
+
+class CrawlerScheduler {
+  constructor() {
+    InitGitConfig();
+
+    this.Start();
+  }
+
+  private async Start() {
+    console.log("Start Crawler");
+    const validatorList = await getValidatorList();
+    if (validatorList.length === 0) {
+      console.log("validatorList length is zero");
+      this.Scheduler();
+    }
+
+    const identityInfos = organizeIdentity(validatorList);
+    if (identityInfos.length === 0) {
+      console.log("identityInfos length is zero");
+      this.Scheduler();
+    }
+
+    const imageUrlByIdentityInfos = await getImageUrlByIdentityInfos(identityInfos);
+    const newProfileJsonString = JSON.stringify(imageUrlByIdentityInfos);
+
+    const oldIdentityJsonString = getIdentityFile();
+    let oldProfileJsonString = "";
+
+    if (oldIdentityJsonString !== "") {
+      const oldProfileInfo: IdentityFileInfo = JSON.parse(oldIdentityJsonString);
+      oldProfileJsonString = JSON.stringify(oldProfileInfo.profileInfos);
+    }
+
+    const compareResult = compareIdentityInfo(newProfileJsonString, oldProfileJsonString);
+
+    console.log("End crawler");
+
+    if (!compareResult) {
+      const dateTime = new Date().getTime();
+      const fileInfo: IdentityFileInfo = {
+        profileInfos: imageUrlByIdentityInfos,
+        lastUpdatedTime: dateTime
+      };
+
+      writeIdentityFile(JSON.stringify(fileInfo));
+
+      gitPushProcess((isSuccess, comment) => {
+        console.log(comment);
+        this.Scheduler();
+      });
+    } else {
+      this.Scheduler();
+    }
+  }
+
+  private async Scheduler() {
+    setTimeout(async () => {
+      await this.Start();
+    }, 5000);
+  }
+}
+
+new CrawlerScheduler();

src/interfaces/common.ts

@@ -0,0 +1,14 @@
+export interface ShortDescription {
+  operatorAddress: string,
+  identity: string
+};
+
+export interface ValidatorProfileInfo {
+  operatorAddress: string,
+  url: string
+};
+
+export interface IdentityFileInfo {
+  profileInfos: ValidatorProfileInfo[],
+  lastUpdatedTime: number
+};

src/utils/axios.ts

@@ -0,0 +1,16 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { KEYBASE_PATH, KEYBASE_URL } from '../config';
+
+const axiosConfig: AxiosRequestConfig = { baseURL: KEYBASE_URL };
+const axiosClient = axios.create(axiosConfig);
+
+export const getAxios = async (identity: string, valOperAddr: string) => {
+  const response = await axiosClient.get(KEYBASE_PATH + identity);
+  try {
+    const url = response.data['them'][0]['pictures']['primary']['url'];
+    return url;
+  } catch (e) {
+    console.log(valOperAddr);
+    return "";
+  }
+}

src/utils/bashCmd.ts

@@ -0,0 +1,14 @@
+import { exec } from "child_process";
+
+export const executeCommand = (cmd: string, callback: (isSuccess: boolean) => void) => {
+  exec(cmd, (err, stdout, stderr) => {
+    if (err) {
+      console.log(stderr);
+      callback(false);
+    }
+
+    console.log(stdout);
+
+    callback(true);
+  });
+}

src/utils/crawlerUtil.ts

@@ -0,0 +1,75 @@
+import { FirmaConfig, FirmaSDK, FirmaUtil, ValidatorDataType } from "@firmachain/firma-js";
+import { ShortDescription, ValidatorProfileInfo } from "../interfaces/common";
+import { getAxios } from "./axios";
+
+export const getValidatorList = async () => {
+  try {
+    const firma = new FirmaSDK(FirmaConfig.MainNetConfig);
+    const validatorList = await firma.Staking.getValidatorList();
+
+    let dataList: ValidatorDataType[] = validatorList.dataList;
+    let nextKey: string = validatorList.pagination.next_key;
+
+    while (nextKey !== null) {
+      const nextValidatorList = await firma.Staking.getValidatorList(nextKey);
+      const nextDataList = nextValidatorList.dataList;
+      nextKey = nextValidatorList.pagination.next_key;
+
+      dataList.push(...nextDataList);
+    }
+
+    return dataList;
+  } catch (e) {
+    console.log(e);
+    return [];
+  }
+}
+
+export const organizeIdentity = (validatorList: ValidatorDataType[]) => {
+  let IdentityInfos: ShortDescription[] = [];
+
+  for (let i = 0; i < validatorList.length; i++) {
+    const currentValidator = validatorList[i];
+
+    try {
+      const valOperAddr = currentValidator.operator_address;
+      const identity = currentValidator.description.identity;
+
+      if (identity !== "") {
+        IdentityInfos.push({ operatorAddress: valOperAddr, identity: identity });
+      }
+    } catch (e) {
+      console.log(e);
+      console.log(`${currentValidator.operator_address} - Abnormalities in value <- Identity ->.`);
+    }
+  }
+
+  return IdentityInfos;
+}
+
+export const getImageUrlByIdentityInfos = async (identityInfos: ShortDescription[]) => {
+  let validatorProfileInfos: ValidatorProfileInfo[] = [];
+
+  for (let i = 0; i < identityInfos.length; i++) {
+    const currentInfo = identityInfos[i];
+    const imageUrl = await getAxios(currentInfo.identity, currentInfo.operatorAddress);
+
+    validatorProfileInfos.push({
+      operatorAddress: currentInfo.operatorAddress,
+      url: imageUrl
+    })
+  }
+
+  return validatorProfileInfos;
+}
+
+export const compareIdentityInfo = (newString: string, oldString: string) => {
+  const newFileHash = FirmaUtil.getHashFromString(newString);
+  const oldFileHash = FirmaUtil.getHashFromString(oldString);
+
+  if (newFileHash === oldFileHash) {
+    return true;
+  }
+
+  return false;
+}

src/utils/file.ts

@@ -0,0 +1,21 @@
+
+import { IDENTITY_FILE_NAME, PUBLIC_PATH } from '../config';
+import * as fs from 'fs';
+
+export const writeIdentityFile = (jsonString: string) => {
+  try {
+    fs.writeFileSync(PUBLIC_PATH + IDENTITY_FILE_NAME, jsonString, 'utf-8');
+    console.log('[COMPLETED] write file');
+  } catch (e) {
+    console.log('[FAILED] write file');
+  }
+}
+
+export const getIdentityFile = () => {
+  try {
+    const identityInfo = fs.readFileSync(PUBLIC_PATH + IDENTITY_FILE_NAME, 'utf-8');
+    return identityInfo;
+  } catch (e) {
+    return "";
+  }
+}

src/utils/gitUtil.ts

@@ -0,0 +1,52 @@
+import {
+  GIT_ADD_FILE_NAME, GIT_COMMIT_FILE_NAME, GIT_CONFIG_FILE_NAME, GIT_PULL_FILE_NAME, GIT_PUSH_FILE_NAME
+} from "../config";
+import { executeCommand } from "./bashCmd";
+
+export const InitGitConfig = () => {
+  executeCommand(GIT_CONFIG_FILE_NAME, (isSuccess) => {
+    if (isSuccess) {
+      console.log('[SUCCESS] GIT CONFIG SET UP');
+    }
+    return isSuccess;
+  });
+};
+
+export const gitPushProcess = (callback: (isSuccess: boolean, comment: string) => void) => {
+  executeCommand(GIT_PULL_FILE_NAME, (pullResult) => {
+    if (pullResult) {
+      console.log('[SUCCESS] GIT PULL');
+
+      executeCommand(GIT_ADD_FILE_NAME, (addResult) => {
+        if (addResult) {
+          console.log('[SUCCESS] GIT ADD FILE');
+
+          executeCommand(GIT_COMMIT_FILE_NAME, (commitResult) => {
+            if (commitResult) {
+              console.log('[SUCCESS] GIT COMMIT');
+
+              executeCommand(GIT_PUSH_FILE_NAME, (pushResult) => {
+                if (pushResult) {
+                  console.log('[SUCCESS] GIT PUSH');
+                  callback(pushResult, "process completed");
+                } else {
+                  console.log('[FAILED] GIT PUSH');
+                  callback(pushResult, "process failed");
+                }
+              });
+            } else {
+              console.log('[FAILED] GIT COMMIT');
+              callback(commitResult, "process failed");
+            }
+          });
+        } else {
+          console.log('[FAILED] GIT ADD FILE');
+          callback(addResult, "process failed");
+        }
+      });
+    } else {
+      console.log('[FAILED] GIT PULL');
+      callback(pullResult, "process failed");
+    }
+  });
+};

tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "strictPropertyInitialization": false,
+    "experimentalDecorators": true,
+    "target": "es5",
+    "module": "commonjs",
+    "baseUrl": "src",
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "lib": ["es6", "dom", "esnext.asynciterable"],
+    "downlevelIteration": true,
+    "strict": true
+  },
+  "include": ["src/**/*.ts", ".env"],
+  "exclude": ["node_modules", "test"]
+}