TW-1459 Implement a minimal demo of categorizing web sites with Cyren API

.env.dist

@@ -10,3 +10,4 @@ THREE_ROUTE_API_AUTH_TOKEN=
 REDIS_URL=
 ADMIN_USERNAME=
 ADMIN_PASSWORD=
+CTWSD_CONTAINER_ADDRESS=

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14
+FROM node:20
 WORKDIR /usr/src/app
 COPY package.json yarn.lock ./
 RUN yarn

Dockerfile.ctwsd

@@ -0,0 +1,12 @@
+FROM --platform=linux/amd64 ubuntu:22.04
+WORKDIR /usr/src/app
+ARG CTWSD_FTP_URL
+ARG CTWSD_FTP_USER
+ARG CTWSD_FTP_PASS
+ARG CTWSD_SERVER_ADDRESS
+ARG CTWSD_LICENSE_KEY
+COPY ./setup_ctwsd.sh ./ctwsd-config/* /usr/src/app/
+RUN CTWSD_FTP_URL=${CTWSD_FTP_URL} CTWSD_FTP_USER=${CTWSD_FTP_USER} CTWSD_FTP_PASS=${CTWSD_FTP_PASS} \
+  CTWSD_LICENSE_KEY=${CTWSD_LICENSE_KEY} CTWSD_SERVER_ADDRESS=${CTWSD_SERVER_ADDRESS} ./setup_ctwsd.sh
+EXPOSE 8080
+CMD [ "./ctwsd/bin/ctwsd", "-i", "-c", "./ctwsd/bin/ctwsd.conf" ]

README.md

@@ -26,11 +26,11 @@ Use `yarn run build` for building.
 
 ## Building and running using Docker
 
-Follow the instructions on [Building your image](https://nodejs.org/en/docs/guides/nodejs-docker-webapp/#building-your-image) and [Running the image](https://nodejs.org/en/docs/guides/nodejs-docker-webapp/#run-the-image). Change the port number in both `.env` and `Dockerfile` if the webapp should be run on another port than 3000.
+Follow the instructions on [Building your image](https://nodejs.org/en/docs/guides/nodejs-docker-webapp/#building-your-image) and [Running the image](https://nodejs.org/en/docs/guides/nodejs-docker-webapp/#run-the-image). Change the port number in both `.env` and `Dockerfile` if the webapp should be run on another port than 3000. You also need to build and run a container with `ctwsd`, see "Building and running `ctwsd` host".
 
 ## Running with pm2
 
-You can run the built backend using `pm2 restart templewallet-backend.json`.
+You can run the built backend using `pm2 restart templewallet-backend.json`. You also need to build and run a container with `ctwsd`, see "Building and running `ctwsd` host".
 
 ## Upstreaming using nginx
 
@@ -67,3 +67,19 @@ location /api/exchange-rates/ {
 ```
 
 Replace 3000 with the respective port number if the backend is listening on a different one. Restart nginx using `sudo systemctl restart nginx` after changes are saved.
+
+## Building and running `ctwsd` host
+
+1. Build the image with command
+```
+docker build \
+  --build-arg CTWSD_FTP_URL=<URL to ctwsd archive for Ubuntu 22.04> \
+  --build-arg CTWSD_FTP_USER=<FTP server username> \
+  --build-arg CTWSD_FTP_PASS=<FTP server password> \
+  --build-arg CTWSD_SERVER_ADDRESS=<Cyren Datacenter server address> \
+  --build-arg CTWSD_LICENSE_KEY=<URLF License Key>:<OEM id> \
+  --file Dockerfile.ctwsd .
+```
+OEM id is an arbitrary alphanumeric string with up to 35 characters.
+2. Get image ID with command `docker image ls`
+3. Start a container with the new image using command `docker container run -d -p <destination port>:8080 <image ID>`

ctwsd-config/CustomCategoryDefinition.ini

@@ -0,0 +1,3 @@
+[501]
+Name = Crypto
+Description = Cryptocurrencies, blockchain, and so on

ctwsd-config/CustomCategoryIndex.idx

@@ -0,0 +1,2 @@
+/usr/src/app/cc_snapshot_1.txt
+/usr/src/app/cc_delta_1

ctwsd-config/cc_snapshot_1.txt

@@ -0,0 +1,37 @@
+ieh	ether|bsc|blockchain|bitcoin|btc|^(www\.)?dex|litecoin|ripple|crypto|solana|altcoin|stablecoin|arbitrum|xtz|dapp|airdrop	501
+i	autofaucet.dutchycorp.space	501
+i	basescan.org	501
+i	bitdegree.org	501
+i	blockchair.com	501
+i	blockworks.co	501
+i	coin360.com	501
+i	coinarbitragebot.com	501
+i	coincarp.com	501
+i	coincheckup.com	501
+i	coincodex.com	501
+i	coincost.net	501
+i	coinedition.com	501
+i	coingape.com	501
+i	coingolive.com	501
+i	coinlisting.info	501
+i	coinmarketcap.com	501
+i	coinpaprika.com	501
+i	coinpedia.org	501
+i	coinspeaker.com	501
+i	cointelegraph.com	501
+i	cryptfaucet.com	501
+i	decrypt.co	501
+i	earn-pepe.com	501
+i	eth-converter.com	501
+i	nft.news	501
+i	polygonscan.com	501
+i	rollercoin.com	501
+i	solscan.io	501
+i	theblock.co	501
+i	thecoinrise.com	501
+i	tzkt.io	501
+i	u.today	501
+i	walletinvestor.com	501
+i	www.coindesk.com	501
+i	www.coingecko.com	501
+i	www.geckoterminal.com	501

package.json

@@ -33,6 +33,7 @@
     "pino-http": "^5.5.0",
     "pino-pretty": "^4.7.1",
     "qs": "^6.10.3",
+    "rate-limiter-flexible": "^5.0.3",
     "semaphore": "^1.1.0",
     "semver": "^7.6.0",
     "swagger-jsdoc": "^6.2.8",

setup_ctwsd.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+CC_CONFDIR=/usr/src/app
+ESCAPED_CC_CONFDIR=$(echo $CC_CONFDIR | sed 's/\//\\\//g')
+DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y wget
+wget $CTWSD_FTP_URL --user=$CTWSD_FTP_USER --password=$CTWSD_FTP_PASS --no-check-certificate -O ./ctwsd.tar.gz
+tar -xzf ctwsd.tar.gz
+rm ctwsd.tar.gz
+mv ./ctwsd* ./ctwsd
+
+CONF_FILE_PATH=./ctwsd/bin/ctwsd.conf
+CC_CACHE_MAX_ENTRIES=10000
+sed -i  "s/ServerAddress \= xxxxxxxxx/ServerAddress = ${CTWSD_SERVER_ADDRESS}/" $CONF_FILE_PATH
+sed -i  "s/LicenseKey \= xxxxxxxxx/LicenseKey = ${CTWSD_LICENSE_KEY}/" $CONF_FILE_PATH
+sed -i  "s/#LocalCustomCategories-Enabled=0/LocalCustomCategories-Enabled=1/" $CONF_FILE_PATH
+sed -i  "s/#CustomCategoriesCacheMaxEntries=10000/CustomCategoriesCacheMaxEntries=${CC_CACHE_MAX_ENTRIES}/" \
+  $CONF_FILE_PATH
+sed -i  "s/#LocalCustomCategories-Uri=/LocalCustomCategories-Uri=${ESCAPED_CC_CONFDIR}\/CustomCategoryIndex.idx/" \
+  $CONF_FILE_PATH
+sed -i  "s/#LocalCustomCategoriesDefinitionFileURI=/LocalCustomCategoriesDefinitionFileURI=\
+${ESCAPED_CC_CONFDIR}\/CustomCategoryDefinition.ini/" $CONF_FILE_PATH

src/config.ts

@@ -13,7 +13,8 @@ export const EnvVars = {
   REDIS_URL: getEnv('REDIS_URL'),
   ADMIN_USERNAME: getEnv('ADMIN_USERNAME'),
   ADMIN_PASSWORD: getEnv('ADMIN_PASSWORD'),
-  COVALENT_API_KEY: getEnv('COVALENT_API_KEY')
+  COVALENT_API_KEY: getEnv('COVALENT_API_KEY'),
+  CTWSD_CONTAINER_ADDRESS: getEnv('CTWSD_CONTAINER_ADDRESS')
 };
 
 for (const name in EnvVars) {

src/index.ts

@@ -30,11 +30,14 @@
 import { getAliceBobOrderInfo } from './utils/alice-bob/get-alice-bob-order-info';
 import { getAliceBobPairInfo } from './utils/alice-bob/get-alice-bob-pair-info';
 import { getAliceBobPairsInfo } from './utils/alice-bob/get-alice-bob-pairs-info';
+import { getSiteCategories } from './utils/cyren-api';
 import { CodedError } from './utils/errors';
+import { withCodedExceptionHandler, withQueryParamsValidation } from './utils/express-helpers';
 import { coinGeckoTokens } from './utils/gecko-tokens';
 import { getExternalApiErrorPayload, isDefined, isNonEmptyString } from './utils/helpers';
 import logger from './utils/logger';
 import { getSignedMoonPayUrl } from './utils/moonpay/get-signed-moonpay-url';
+import { adCategoryQueryParamsSchema } from './utils/schemas';
 import { getSigningNonce } from './utils/signing-nonce';
 import SingleQueryDataProvider from './utils/SingleQueryDataProvider';
 import { tezExchangeRateProvider } from './utils/tezos';
@@ -66,6 +69,7 @@
 app.use(pinoHttp(PINO_LOGGER));
 app.use(cors());
 app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
 
 const androidApp = firebaseAdmin.initializeApp(
   {
@@ -156,7 +160,7 @@
     await redisClient.lpush('notifications', JSON.stringify(newNotification));
 
     res.status(200).send({ message: 'Notification added successfully', notification: newNotification });
   } catch (error: any) {
     res.status(500).send({ error: error.message });
   }
 });
@@ -335,6 +339,17 @@
 
 app.use('/api/slise-ad-rules', adRulesRouter);
 
+app.get(
+  '/api/get-ad-category',
+  withCodedExceptionHandler(
+    withQueryParamsValidation(adCategoryQueryParamsSchema, async (req, res) => {
+      const categories = await getSiteCategories(req.query.url);
+
+      res.status(200).send(categories);
+    })
+  )
+);
+
 app.use('/api/evm', evmRouter);
 
 app.post('/api/magic-square-quest/start', async (req, res) => {
@@ -342,7 +357,7 @@
     await startMagicSquareQuest(req.body);
 
     res.status(200).send({ message: 'Quest successfully started' });
   } catch (error: any) {
     console.error(error);
 
     if (error instanceof CodedError) {
@@ -356,7 +371,7 @@
 app.get('/api/magic-square-quest/participants', basicAuth, async (req, res) => {
   try {
     res.status(200).send(await getMagicSquareQuestParticipants());
   } catch (error: any) {
     console.error(error);
 
     if (error instanceof CodedError) {
@@ -370,10 +385,10 @@
 app.get('/api/signing-nonce', (req, res) => {
   try {
     const pkh = req.query.pkh;
     if (!pkh || typeof pkh !== 'string') throw new Error('PKH is not a string');
 
     res.status(200).send(getSigningNonce(pkh));
   } catch (error: any) {
     console.error(error);
 
     if (error instanceof CodedError) {

src/utils/MutexProtectedData.ts

@@ -7,8 +7,8 @@ export default class MutexProtectedData<T> {
     this.mutex = new PromisifiedSemaphore();
   }
 
-  exec(task: () => void | Promise<void>) {
-    return this.mutex.exec(task);
+  exec<U = void>(task: (data: T, setData: (value: T) => void) => U | Promise<U>) {
+    return this.mutex.exec(() => task(this.data, value => void (this.data = value)));
   }
 
   setData(newData: T) {

src/utils/PromisifiedSemaphore.ts

@@ -15,12 +15,12 @@ export default class PromisifiedSemaphore {
     return this.semaphore.available(n);
   }
 
-  exec(task: () => void | Promise<void>, n = 1) {
-    return new Promise<void>((resolve, reject) => {
+  exec<T>(task: () => T | Promise<T>, n = 1) {
+    return new Promise<T>((resolve, reject) => {
       this.semaphore.take(n, async () => {
         try {
-          await task();
-          resolve();
+          const value = await task();
+          resolve(value);
         } catch (e) {
           reject(e);
         } finally {

src/utils/SingleQueryDataProvider.ts

@@ -42,7 +42,9 @@ export default class SingleQueryDataProvider<T> {
   ) {
     this.fetchMutex = new PromisifiedSemaphore();
     this.readyMutex = new PromisifiedSemaphore();
-    this.state = new MutexProtectedData({ error: new Error('This error should not be displayed') });
+    this.state = new MutexProtectedData<SingleQueryDataProviderState<T>>({
+      error: new Error('This error should not be displayed')
+    });
     this.init();
   }