Code Review

.dockerignore

@@ -0,0 +1,11 @@
+.git/
+.ipynb_checkpoints/
+.__pycache__/
+.env
+.gitattributes
+.gitignore
+Pipfile
+Pipfile.lock
+todo.md
+Dockerfile
+.dockerignore

.gitignore

@@ -1,3 +1,6 @@
+# Program speicifc
+logs/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.7-stretch
+COPY . /app
+WORKDIR /app
+RUN mkdir logs
+EXPOSE 80
+RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list
+RUN apt update -y
+RUN apt install -y build-essential curl cron
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN rustup update
+RUN pip3 install -r requirements.txt
+ADD ./docker-scripts/crontab /etc/cron.d/simple-cron
+RUN chmod +x ./docker-scripts/event_master.sh 
+RUN chmod +x ./docker-scripts/refresh_master.sh
+RUN chmod +x ./docker-scripts/start_server.sh
+RUN chmod +x ./docker-scripts/store_volume.sh
+RUN chmod +x ./docker-scripts/setup.sh
+RUN chmod 0644 /etc/cron.d/simple-cron
+RUN touch /var/log/cron.log
+CMD ./docker-scripts/setup.sh
+# CMD cron && ./docker-scripts/refresh_master.sh && ./docker-scripts/store_volume.sh && tail -F /var/log/cron.log

Pipfile

@@ -0,0 +1,20 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+gevent = "*"
+web3 = "*"
+requests = "*"
+ipykernel = "*"
+numpy = "*"
+python-dotenv = "*"
+pymongo = {extras = ["srv"], version = "*"}
+discord = "*"
+fastapi = {extras = ["all"], version = "*"}
+
+[dev-packages]
+
+[requires]
+python_version = "3.7"

README.md

@@ -0,0 +1,67 @@
+# Machine setup
+
+Expose port 80 on your machine for outgoing TCP requests
+
+# Method 1 - Running with Docker
+
+Pass your environment variables after building using the following command. 
+
+```
+cd
+git clone [email protected]:mycelium-ethereum/punk-offerbook.git
+cd punk-offerbook
+docker build . -t ps
+docker run -d --name ps_container -e MONGO_URL="" -e ETH_HTTP_URL="" -e ETHERSCAN_KEY="" -p 80:80 ps
+```
+
+--- 
+
+# Method 2
+## Installation
+
+```
+cd
+git clone [email protected]:mycelium-ethereum/punk-offerbook.git
+cd punk-offerbook
+pip3 install -r requirements.txt
+```
+
+## Setup environment
+
+Create a .env file with the following variables\
+The DISCORD_WEBHOOK env variable is optional. Set to an empty string if you don't want to see server logs on discord.
+```
+ETH_HTTP_URL = "XXX"
+MONGO_URL = "XXX"
+DISCORD_WEBHOOK = "XXX"
+```
+
+## Install cron
+
+Replace < username > with your server username.
+```
+*/55 * * * * /home/<username>/punk-offerbook/shell-scripts/refresh_master.sh
+* * * * * /usr/bin/flock -n /tmp/punk_offerbook_events.lockfile /home/<username>/punk-offerbook/shell-scripts/event_master.sh
+* * * * * /usr/bin/flock -n /tmp/punk_offerbook_server.lockfile /home/<username>/punk-offerbook/shell-scripts/start_server.sh
+```
+
+## Data preparation
+```
+cd
+./punk-offerbook/shell-scripts/event_master.sh &
+./punk-offerbook/shell-scripts/refresh_master.sh
+```
+
+## Starting the server
+By default the server will serve data through port 3400. This can be modified in the start_server.sh file.\
+To run the server on port 3400, execute the following command:-
+```
+uvicorn floor_price:app --host 0.0.0.0 --port 3400
+```
+
+## Usage
+
+Replace < external_ip > with your External IP address.
+```
+curl -X GET -H "content-type:application/json" "http://<external_ip>:3400/punkfloor"
+```

abis/CryptoPunksMarket.json

@@ -0,0 +1 @@
+{"abi": [{"constant": true, "inputs": [], "name": "name", "outputs": [{"name": "", "type": "string"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [{"name": "", "type": "uint256"}], "name": "punksOfferedForSale", "outputs": [{"name": "isForSale", "type": "bool"}, {"name": "punkIndex", "type": "uint256"}, {"name": "seller", "type": "address"}, {"name": "minValue", "type": "uint256"}, {"name": "onlySellTo", "type": "address"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}], "name": "enterBidForPunk", "outputs": [], "payable": true, "type": "function"}, {"constant": true, "inputs": [], "name": "totalSupply", "outputs": [{"name": "", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}, {"name": "minPrice", "type": "uint256"}], "name": "acceptBidForPunk", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "decimals", "outputs": [{"name": "", "type": "uint8"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "addresses", "type": "address[]"}, {"name": "indices", "type": "uint256[]"}], "name": "setInitialOwners", "outputs": [], "payable": false, "type": "function"}, {"constant": false, "inputs": [], "name": "withdraw", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "imageHash", "outputs": [{"name": "", "type": "string"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "nextPunkIndexToAssign", "outputs": [{"name": "", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [{"name": "", "type": "uint256"}], "name": "punkIndexToAddress", "outputs": [{"name": "", "type": "address"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "standard", "outputs": [{"name": "", "type": "string"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [{"name": "", "type": "uint256"}], "name": "punkBids", "outputs": [{"name": "hasBid", "type": "bool"}, {"name": "punkIndex", "type": "uint256"}, {"name": "bidder", "type": "address"}, {"name": "value", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": true, "inputs": [{"name": "", "type": "address"}], "name": "balanceOf", "outputs": [{"name": "", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [], "name": "allInitialOwnersAssigned", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "allPunksAssigned", "outputs": [{"name": "", "type": "bool"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}], "name": "buyPunk", "outputs": [], "payable": true, "type": "function"}, {"constant": false, "inputs": [{"name": "to", "type": "address"}, {"name": "punkIndex", "type": "uint256"}], "name": "transferPunk", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "symbol", "outputs": [{"name": "", "type": "string"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}], "name": "withdrawBidForPunk", "outputs": [], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "to", "type": "address"}, {"name": "punkIndex", "type": "uint256"}], "name": "setInitialOwner", "outputs": [], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}, {"name": "minSalePriceInWei", "type": "uint256"}, {"name": "toAddress", "type": "address"}], "name": "offerPunkForSaleToAddress", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [], "name": "punksRemainingToAssign", "outputs": [{"name": "", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}, {"name": "minSalePriceInWei", "type": "uint256"}], "name": "offerPunkForSale", "outputs": [], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}], "name": "getPunk", "outputs": [], "payable": false, "type": "function"}, {"constant": true, "inputs": [{"name": "", "type": "address"}], "name": "pendingWithdrawals", "outputs": [{"name": "", "type": "uint256"}], "payable": false, "type": "function"}, {"constant": false, "inputs": [{"name": "punkIndex", "type": "uint256"}], "name": "punkNoLongerForSale", "outputs": [], "payable": false, "type": "function"}, {"inputs": [], "payable": true, "type": "constructor"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "to", "type": "address"}, {"indexed": false, "name": "punkIndex", "type": "uint256"}], "name": "Assign", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "from", "type": "address"}, {"indexed": true, "name": "to", "type": "address"}, {"indexed": false, "name": "value", "type": "uint256"}], "name": "Transfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "from", "type": "address"}, {"indexed": true, "name": "to", "type": "address"}, {"indexed": false, "name": "punkIndex", "type": "uint256"}], "name": "PunkTransfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "punkIndex", "type": "uint256"}, {"indexed": false, "name": "minValue", "type": "uint256"}, {"indexed": true, "name": "toAddress", "type": "address"}], "name": "PunkOffered", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "punkIndex", "type": "uint256"}, {"indexed": false, "name": "value", "type": "uint256"}, {"indexed": true, "name": "fromAddress", "type": "address"}], "name": "PunkBidEntered", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "punkIndex", "type": "uint256"}, {"indexed": false, "name": "value", "type": "uint256"}, {"indexed": true, "name": "fromAddress", "type": "address"}], "name": "PunkBidWithdrawn", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "punkIndex", "type": "uint256"}, {"indexed": false, "name": "value", "type": "uint256"}, {"indexed": true, "name": "fromAddress", "type": "address"}, {"indexed": true, "name": "toAddress", "type": "address"}], "name": "PunkBought", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "name": "punkIndex", "type": "uint256"}], "name": "PunkNoLongerForSale", "type": "event"}]}

app.py

@@ -0,0 +1,28 @@
+from utils import *
+from fastapi import FastAPI
+
+app = FastAPI()
+logger = setup_custom_logger('root')
+setup_file_logger('server', logger)
+
+@app.get("/punkfloor")
+async def root():
+    try:
+        # get data from database
+        cryptopunks_offers = parse_db_offers(mongo.get_all_offers())
+        cryptopunks_txs = parse_db_txs(mongo.get_transactions(timedelta(days=7)))
+
+        # check if data is ready 
+        if len(cryptopunks_offers) != 10000:
+            return {'result': 0, 'data': {'error': 'Data is not ready. Should be ready in <1 hr.'}}
+
+        # extract floor price and volume traded 
+        cryptopunks_offers = [offer for offer in cryptopunks_offers if offer.is_valid]
+        cryptopunks_offers.sort(key=lambda x: x.min_value, reverse=False)
+        volume_traded = sum(tx['value'] for tx in cryptopunks_txs) / 7
+
+        return {'result': 1, 'data': {'price': cryptopunks_offers[0].min_value, 'volume': volume_traded}}
+
+    except Exception as e:
+        logger.error(f"Error in server - {e}")
+        return {'result': 0, 'data': {'error': 'x should be greater than or equal to 1'}}

client/Mongo.py

@@ -0,0 +1,45 @@
+import os
+import ssl
+import logging
+from typing import List, Dict
+from pymongo import MongoClient
+from datetime import datetime, timedelta
+
+class Mongo():
+    def __init__(self):
+        self.logger = logging.getLogger('root')
+        self.client = MongoClient(os.environ.get("MONGO_URL"), ssl=True, ssl_cert_reqs=ssl.CERT_NONE, readPreference='nearest')
+        self.cryptopunks_offerbook  = self.client.nft_offer_books.cryptopunks
+        self.cryptopunks_transactions = self.client.nft_transactions.cryptopunks
+
+    def transaction_present(self, tx_hash: str) -> bool:
+        return self.cryptopunks_transactions.find_one({'hash': tx_hash}) != None
+
+    def get_transactions(self, timedelta: timedelta) -> List[Dict]:
+        self.logger.info("Getting transactions...")
+        return list(self.cryptopunks_transactions.find({
+            'ts': {'$gte': datetime.utcnow() - timedelta}
+        }))
+
+    def get_all_offers(self) -> List[Dict]:
+        self.logger.info("Getting all offers...")
+        return list(self.cryptopunks_offerbook.find({}))
+
+    def get_offer(self, punk_index: int) -> Dict:
+        self.logger.info(f"Getting offer for {punk_index}...")
+        return self.cryptopunks_offerbook.find_one({'punk_index': punk_index})
+
+    def insert_offer(self, offer: Dict):
+        self.logger.info(f"Inserting offer for {offer['punk_index']}...")
+        self.cryptopunks_offerbook.insert_one(offer)
+
+    def update_offer(self, offer: Dict):
+        self.logger.info(f"Updating offer for {offer['punk_index']}...")
+        self.cryptopunks_offerbook.update_one(
+            {'punk_index': offer['punk_index']},
+            {'$set': offer},
+            upsert=False
+        )
+
+    def offer_present(self, punk_index: int) -> bool:
+        return self.cryptopunks_offerbook.find_one({'punk_index': punk_index}) != None

client/Opensea.py

@@ -0,0 +1,36 @@
+import requests
+from typing import List, Dict
+
+class Opensea:
+    BASE_URL = "https://api.opensea.io/wyvern/v1"
+    OFFSET = 50
+
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key
+        if self.api_key is not None:
+            self.headers = {
+                "Accept": "application/json",
+                "X-API-KEY": self.api_key,
+            }
+        else:
+            self.headers = {"Accept": "application/json"}
+
+    def get_offers(self, address: str, id_string: str, offset: int = 0) -> List[Dict]:
+        endpoint = "/orders"
+        url = (
+            f"{self.BASE_URL + endpoint}?asset_contract_address={address}&is_english=false"
+            f"&bundled=false&include_bundled=false&include_invalid=false&{id_string}"
+            f"&limit=50&offset={offset}")
+        response = requests.request("GET", url, headers=self.headers)
+        return response.json()['orders']
+
+    def get_all_offers(self, address: str, id_string: str) -> List[Dict]:
+        offset = 0
+        offers = []
+        flag = True
+        while flag:
+            _offers = self.get_offers(address, id_string, offset)
+            offers += _offers
+            if len(_offers) < self.OFFSET: flag = False
+            else: offset += self.OFFSET
+        return offers

client/Postgres.py

@@ -0,0 +1,3 @@
+# class Postgres:
+#     def __init__(self):
+#         self.client = psycopg2.connect("dbname=suppliers user=postgres password=postgres")

client/Webhook.py

@@ -0,0 +1,6 @@
+import os
+from discord import Webhook, RequestsWebhookAdapter
+
+URL = os.getenv('DISCORD_WEBHOOK')
+if URL is not None and URL != "": webhook = Webhook.from_url(URL, adapter=RequestsWebhookAdapter())
+else: webhook = None

client/__init__.py

@@ -0,0 +1,22 @@
+from dotenv import load_dotenv
+load_dotenv();
+
+import os
+import json
+import settings
+from web3 import Web3
+from client.Mongo import Mongo
+from client.Webhook import webhook
+from client.Opensea import Opensea
+
+def get_raw_abis(abi_paths):
+    raw_abis = {}
+    for abi_key, abi_path in abi_paths.items():
+        with open(abi_path, "r") as f:
+            raw_abis[abi_key] = json.loads(f.read())['abi']
+    return raw_abis
+
+abis = get_raw_abis(settings.ABI_PATHS)
+web3 = Web3(Web3.HTTPProvider(os.environ.get("ETH_HTTP_URL")))
+mongo = Mongo()
+opensea = Opensea()

docker-scripts/crontab

@@ -0,0 +1,4 @@
+*/15 * * * * root /app/docker-scripts/refresh_master.sh
+* * * * * root /usr/bin/flock -n /tmp/punk_offerbook_events.lockfile /app/docker-scripts/event_master.sh
+* * * * * root /usr/bin/flock -n /tmp/punk_offerbook_server.lockfile /app/docker-scripts/start_server.sh
+# empty line 

docker-scripts/event_master.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$(date): Executing event master" >> /var/log/cron.log 2>&1
+cd /app
+/usr/local/bin/python3 event_watcher.py

docker-scripts/refresh_master.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$(date): Executing refresh master" >> /var/log/cron.log 2>&1
+cd /app
+/usr/local/bin/python3 refresh_offers.py

docker-scripts/setup.sh

@@ -0,0 +1,7 @@
+# !/usr/bin/env bash
+
+printenv | grep -v "no_proxy" >> /etc/environment
+cron
+/app/docker-scripts/store_volume.sh
+/app/docker-scripts/refresh_master.sh
+tail -F /var/log/cron.log

docker-scripts/start_server.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$(date): Starting server" >> /var/log/cron.log 2>&1
+cd /app
+/usr/local/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 80

docker-scripts/store_volume.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$(date): Executing store volume" >> /var/log/cron.log 2>&1
+cd /app
+/usr/local/bin/python3 store_volume_data.py

event_watcher.py

@@ -0,0 +1,47 @@
+import sys
+import settings
+from utils import *
+from client import mongo, webhook
+
+def update_handler(event: dict):
+    global cryptopunks
+    event = dict(event)
+    punk_index = event['args']['punkIndex']
+    db_offer = parse_db_offer(mongo.get_offer(punk_index))
+    api_offer = cryptopunks.get_offer(punk_index)
+    if api_offer.ts > db_offer.ts and not api_offer.equals(db_offer):
+        alert(f"Updating offer for PUNK {punk_index}")
+        mongo.update_offer(api_offer.db_parse())
+
+def bought_handler(event: dict):
+    global cryptopunks
+    update_handler(event)
+    event = dict(event)
+    tx_data = {
+        'ts': datetime.utcnow(), 
+        'hash': event['transactionHash'].hex(),
+        'value': str(cryptopunks.get_tx_price(event['transactionHash'].hex()))
+    }
+    if not mongo.transaction_present(tx_data['hash']):
+        alert("Storing transaction...")
+        mongo.cryptopunks_transactions.insert_one(tx_data)
+
+if __name__ == "__main__":
+    logger = setup_custom_logger('root')
+    setup_file_logger('event', logger)
+
+    alert('Starting event watcher now.')
+    logger.info('Starting event watcher now.')
+
+    cryptopunks = Cryptopunks()
+    for event in settings.CRYPTO_PUNKS_EVENTS:
+        if event == 'PunkBought':
+            cryptopunks.start_streaming(event, event_handler=bought_handler)
+        else:
+            cryptopunks.start_streaming(event, event_handler=update_handler)
+
+    while not cryptopunks.errored: time.sleep(1)
+
+    alert(f"Quitting event watcher because of error - {cryptopunks.error}")
+    logger.error(f"Quitting event watcher because of error - {cryptopunks.error}")
+    sys.exit(1)