diff --git a/bitcoinqt-coinjoiner.py b/bitcoinqt-coinjoiner.py new file mode 100644 index 00000000..1d4214b9 --- /dev/null +++ b/bitcoinqt-coinjoiner.py @@ -0,0 +1,265 @@ +#! /usr/bin/env python + +from __future__ import absolute_import + +import socket, json, threading, pprint, re +from optparse import OptionParser + +from joinmarket import Taker, load_program_config, IRCMessageChannel +from joinmarket import validate_address, jm_single, rand_norm_array +from joinmarket import random_nick +from joinmarket import get_log, choose_sweep_orders, choose_orders, \ + weighted_order_choose, debug_dump_object +from joinmarket import BlockchainInterface, BitcoinCoreWallet +from joinmarket.wallet import estimate_tx_fee +from joinmarket.configure import get_p2sh_vbyte, get_p2pk_vbyte +from joinmarket.jsonrpc import JsonRpcConnectionError, JsonRpcError +import bitcoin as btc + +log = get_log() + +def ok_orders(total_fee_pc): + WARNING_THRESHOLD = 0.02 # 2% + if total_fee_pc > WARNING_THRESHOLD: + print('\n'.join(['=' * 60] * 3)) + print('WARNING ' * 6) + print('\n'.join(['=' * 60] * 1)) + print('OFFERED COINJOIN FEE IS UNUSUALLY HIGH. DOUBLE/TRIPLE CHECK.') + print('\n'.join(['=' * 60] * 1)) + print('WARNING ' * 6) + print('\n'.join(['=' * 60] * 3)) + jm_single().debug_silence[0] = True + ret = raw_input('send with these orders? (y/n):')[0] == 'y' + jm_single().debug_silence[0] = False + return ret + +def obtain_utxo_data(txid, index): + try: + txdata = jm_single().bc_interface.rpc('gettransaction', [txid]) + out = btc.deserialize(str(txdata['hex']))['outs'][index] + pprint.pprint(btc.deserialize(str(txdata['hex']))) + addr = btc.script_to_address(out['script'], get_p2pk_vbyte()) + value = out['value'] + return {'address': addr, 'value': value} + except (JsonRpcError, JsonRpcConnectionError) as e: + log.debug('transaction not found, returning') + raise ValueError(repr(e)) + +class BitcoindTaker(Taker): + def __init__(self, msgchan, wallet, options, retry_txid): + super(BitcoindTaker, self).__init__(msgchan) + self.wallet = wallet + self.options = options + self.retry_txid = retry_txid + self.ignored_makers = [] + + def on_welcome(self): + Taker.on_welcome(self) + if self.retry_txid: + threading.Timer(self.options.waittime, + lambda : self.handle_noncj_txid(self.retry_txid)).start() + + def finishcallback(self, coinjointx): + if coinjointx.all_responded: + pushed = coinjointx.self_sign_and_push() + if pushed: + log.debug('created fully signed tx') + return + self.ignored_makers += coinjointx.nonrespondants + log.debug('recreating the tx, ignored_makers=' + str( + self.ignored_makers)) + self.create_tx() + + def bitcoind_choose_orders(self, + cj_amount, + makercount, + nonrespondants=None, + active_nicks=None): + if nonrespondants is None: + nonrespondants = [] + if active_nicks is None: + active_nicks = [] + self.ignored_makers += nonrespondants + orders, total_cj_fee = choose_orders( + self.db, cj_amount, makercount, weighted_order_choose, + self.ignored_makers + active_nicks) + if not orders: + return None, 0 + log.debug('chosen orders to fill ' + str(orders) + ' totalcjfee=' + str( + total_cj_fee)) + total_fee_pc = 1.0 * total_cj_fee / cj_amount + log.debug('coinjoin fee = ' + str(float('%.3g' % ( + 100.0 * total_fee_pc))) + '%') + if not self.options.answeryes: + if not ok_orders(total_fee_pc): + return None, 0 + return orders, total_cj_fee + + def create_tx(self): + if self.change_addr: + log.debug('creating a coinjoin with change') + choose_orders_recover = self.bitcoind_choose_orders + orders, total_cj_fee = self.bitcoind_choose_orders( + self.cjamount, self.maker_count) + if not orders: + log.debug('unable to create coinjoin') + return + else: + log.debug('creating a sweep coinjoin with no change') + orders, self.cjamount, total_cj_fee = choose_sweep_orders( + self.db, input_values, self.txfee, + self.makercount, weighted_order_choose, + ignored_makers=None) + if not orders: + log.debug("Could not find orders to complete transaction.") + return + total_fee_pc = 1.0 * total_cj_fee / self.cj_amount + log.debug(noun + ' coinjoin fee = ' + str(float('%.3g' % ( + 100.0 * total_fee_pc))) + '%') + if not self.options.answeryes: + if not ok_orders(total_fee_pc): + return + log.debug('detected coinjoin amount=' + str(self.cjamount) + + ' cjaddr=' + self.cj_addr + ' change=' + str(self.change_addr)) + self.start_cj(self.wallet, self.cjamount, orders, self.utxos, + self.cj_addr, self.change_addr, self.txfee, + self.finishcallback, choose_orders_recover) + + def handle_noncj_txid(self, txid): + if not re.match('^[0-9a-fA-F]*$', txid): + log.debug('not a txid') + return + try: + txdata = jm_single().bc_interface.rpc('gettransaction', [txid]) + except (JsonRpcError, JsonRpcConnectionError) as e: + log.debug('transaction not found, returning') + return + if txdata['confirmations'] != 0: + log.debug('not an unconfirmed tx, returning') + return + txd = btc.deserialize(str(txdata['hex'])) + if len(txd['outs']) > 2: + log.debug('tx has more outputs than 2, unable to make coinjoin of it') + return + utxo_list = [(ins['outpoint']['hash'], ins['outpoint'] + ['index']) for ins in txd['ins']] + self.utxos = dict([(utxo[0] + ':' + str(utxo[1]), + obtain_utxo_data(*utxo)) for utxo in utxo_list]) + log.debug('utxos = \n' + pprint.pformat(self.utxos)) + input_values = sum([s['value'] for s in self.utxos.values()]) + output_values = sum((o['value'] for o in txd['outs'])) + self.txfee = input_values - output_values + self.maker_count = int(round(rand_norm_array( + self.options.makercountrange[0], + self.options.makercountrange[1], 1)[0])) + log.debug('txfee=' + str(self.txfee) + ' maker_count=' + + str(self.maker_count)) + if len(txd['outs']) == 2: + log.debug('parsing coinjoin with change') + output_addrs = [(btc.script_to_address(o['script'], + get_p2pk_vbyte()), o['value']) for o in txd['outs']] + addr_change = [(a, jm_single().bc_interface.rpc( + 'getreceivedbyaddress', [a[0], 0]) > 0) + for a in output_addrs] + log.debug('addr_change = ' + str(addr_change)) + change = zip(*addr_change)[1] + if change[0] ^ change[1] == False: + log.debug('unable to find which address is change (' + + str(change) + ') returning') + return + cj_out = [ac[0] for ac in addr_change if not ac[1]][0] + self.cj_addr = cj_out[0] + self.cjamount = cj_out[1] + self.change_addr = [ac[0][0] for ac in addr_change if ac[1]][0] + else: + log.debug('parsing sweep coinjoin') + self.cjamount = txd['outs'][0]['value'] + self.cj_addr = btc.script_to_address(txd['outs'][0]['script'], + get_p2pk_vbyte()) + self.change_addr = None + choose_orders_recover = None + ##see the identical code in sendpayment.py for an explaination + est_ins = len(self.utxos) + 3*self.maker_count + log.debug("Estimated ins: "+str(est_ins)) + est_outs = 2*self.makercount + 1 + log.debug("Estimated outs: "+str(est_outs)) + estimated_fee = estimate_tx_fee(est_ins, est_outs) + log.debug("We have a fee estimate: "+str(estimated_fee)) + log.debug("And a requested fee of: "+str(self.maker_count* + self.txfee)) + if estimated_fee > self.maker_count*self.txfee: + #both values are integers; we can ignore small rounding errors + self.txfee = estimated_fee / self.maker_count + self.create_tx() + + def notify_hook(self, requesthandler): + log.debug('notify hook called') + walletnotify = '/walletnotify?' + if requesthandler.path.startswith(walletnotify): + txid = requesthandler.path[len(walletnotify):] + self.handle_noncj_txid(txid) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', + description='Sends a single payment from a given mixing depth of your ' + + + 'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. ' + + + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') + parser.add_option( + '-N', + '--makercountrange', + type='float', + nargs=2, + action='store', + dest='makercountrange', + help= + 'Input the mean and spread of number of makers to use. e.g. 3 1.5 will be a normal distribution ' + 'with mean 3 and standard deveation 1.5 inclusive, default=3 1.5', + default=(3, 1.5)) + parser.add_option('--yes', + action='store_true', + dest='answeryes', + default=False, + help='answer yes to everything') + parser.add_option( + '-w', + '--wait-time', + action='store', + type='float', + dest='waittime', + help='wait time in seconds to allow orders to arrive, default=5', + default=5) + (options, args) = parser.parse_args() + + retry_txid = None + if len(args) > 0: + retry_txid = args[0] + + load_program_config() + #fails if we're not using BitcoinCoreInterface + wallet = BitcoinCoreWallet("") + jm_single().nickname = random_nick() + log.debug('starting joinmarket bitcoind interface') + + irc = IRCMessageChannel(jm_single().nickname) + taker = BitcoindTaker(irc, wallet, options, retry_txid) + + jm_single().bc_interface.notify_hook = taker.notify_hook + jm_single().bc_interface.start_notify_thread() + + try: + log.debug('starting irc') + irc.run() + except: + log.debug('CRASHING, DUMPING EVERYTHING') + debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) + debug_dump_object(taker) + import traceback + log.debug(traceback.format_exc()) + +if __name__ == "__main__": + main() + print('done') diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index 3de500a3..bbb64174 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -406,7 +406,7 @@ def bitcoincore_timeout_callback(uc_called, txout_set, txnotify_fun_list, log.debug('timeoutfun txout_set=\n' + pprint.pformat(txout_set)) timeoutfun(uc_called) -class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): +class NotifyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, request, client_address, base_server): self.btcinterface = base_server.btcinterface self.base_server = base_server @@ -414,8 +414,12 @@ def __init__(self, request, client_address, base_server): self, request, client_address, base_server) def do_HEAD(self): - pages = ('/walletnotify?', '/alertnotify?') + if self.btcinterface.notify_hook: + self.btcinterface.notify_hook(self) + else: + log.debug('no notify hook') + pages = ('/walletnotify?', '/alertnotify?') if self.path.startswith('/walletnotify?'): txid = self.path[len(pages[0]):] if not re.match('^[0-9a-fA-F]*$', txid): @@ -424,7 +428,7 @@ def do_HEAD(self): try: tx = self.btcinterface.rpc('getrawtransaction', [txid]) except (JsonRpcError, JsonRpcConnectionError) as e: - log.debug('transaction not found, probably a conflict') + log.debug('transaction not found, not ours or a conflict') return if not re.match('^[0-9a-fA-F]*$', tx): log.debug('not a txhex') @@ -484,7 +488,8 @@ def do_HEAD(self): log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0]) else: - log.debug('ERROR: This is not a handled URL path. You may want to check your notify URL for typos.') + if not self.btcinterface.notify_hook: + log.debug('ERROR: This is not a handled URL path. You may want to check your notify URL for typos.') request = urllib2.Request('http://localhost:' + str(self.base_server.server_address[1] + 1) + self.path) request.get_method = lambda : 'HEAD' @@ -514,7 +519,7 @@ def run(self): for inc in range(10): hostport = (notify_host, notify_port + inc) try: - httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHeader) + httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHandler) except Exception: continue httpd.btcinterface = self.btcinterface @@ -541,6 +546,7 @@ def __init__(self, jsonRpc, network): raise Exception('wrong network configured') self.notifythread = None + self.notify_hook = None self.txnotify_fun = [] self.wallet_synced = False @@ -548,6 +554,11 @@ def __init__(self, jsonRpc, network): def get_wallet_name(wallet): return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + def start_notify_thread(self): + if not self.notifythread: + self.notifythread = BitcoinCoreNotifyThread(self) + self.notifythread.start() + def rpc(self, method, args): if method not in ['importaddress', 'walletpassphrase']: log.debug('rpc: ' + method + " " + str(args)) @@ -683,9 +694,7 @@ def sync_unspent(self, wallet): def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, timeoutfun=None): - if not self.notifythread: - self.notifythread = BitcoinCoreNotifyThread(self) - self.notifythread.start() + self.start_notify_thread() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) @@ -749,13 +758,13 @@ def __init__(self, jsonRpc): super(RegtestBitcoinCoreInterface, self).__init__(jsonRpc, 'regtest') self.pushtx_failure_prob = 0 self.tick_forward_chain_interval = 2 - self.absurd_fees = False + self.absurd_fees = False def estimate_fee_per_kb(self, N): - if not self.absurd_fees: - return super(RegtestBitcoinCoreInterface, self).estimate_fee_per_kb(N) - else: - return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100 + if not self.absurd_fees: + return super(RegtestBitcoinCoreInterface, self).estimate_fee_per_kb(N) + else: + return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100 def pushtx(self, txhex): if self.pushtx_failure_prob != 0 and random.random() <\ diff --git a/joinmarket/jsonrpc.py b/joinmarket/jsonrpc.py index d12243f6..9e1d80e2 100644 --- a/joinmarket/jsonrpc.py +++ b/joinmarket/jsonrpc.py @@ -114,6 +114,6 @@ def call(self, method, params): raise JsonRpcConnectionError("invalid id returned by query") if response["error"] is not None: - raise JsonRpcError(response["error"]) + raise JsonRpcError(response) return response["result"] diff --git a/test/conftest.py b/test/conftest.py index 9dea36f4..db8f4bfd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,17 +28,34 @@ def pytest_addoption(parser): default='bitcoinrpc', help="the RPC username for your test bitcoin instance (default=bitcoinrpc)") -def teardown(): - #didn't find a stop command in miniircd, so just kill - global miniircd_proc - miniircd_proc.kill() +## a lot of these sleeps should be replaced by rpc calls which detech +## when bitcoin core is ready +def start_bitcoind(additional_args=None): + if not additional_args: + additional_args = [] + #start up regtest blockchain + btc_proc = subprocess.call([bitcoin_path + "bitcoind", "-regtest", + "-daemon", "-conf=" + bitcoin_conf] + additional_args) + time.sleep(20) + #generate blocks + local_command([bitcoin_path + "bitcoin-cli", "-regtest", "-rpcuser=" + + bitcoin_rpcusername, "-rpcpassword=" + bitcoin_rpcpassword, + "generate", "101"]) + time.sleep(10) +def stop_bitcoind(): #shut down bitcoin and remove the regtest dir local_command([bitcoin_path + "bitcoin-cli", "-regtest", "-rpcuser=" + bitcoin_rpcusername, "-rpcpassword=" + bitcoin_rpcpassword, "stop"]) #note, it is better to clean out ~/.bitcoin/regtest but too #dangerous to automate it here perhaps + time.sleep(10) +def teardown(): + #didn't find a stop command in miniircd, so just kill + global miniircd_proc + miniircd_proc.kill() + stop_bitcoind() @pytest.fixture(scope="session", autouse=True) def setup(request): @@ -57,10 +74,5 @@ def setup(request): miniircd_proc = local_command( ["./miniircd/miniircd", "--motd=" + cwd + "/miniircd/testmotd"], bg=True) - #start up regtest blockchain - btc_proc = subprocess.call([bitcoin_path + "bitcoind", "-regtest", - "-daemon", "-conf=" + bitcoin_conf]) - time.sleep(3) - #generate blocks - local_command([bitcoin_path + "bitcoin-cli", "-regtest", "-rpcuser=" + bitcoin_rpcusername, - "-rpcpassword=" + bitcoin_rpcpassword, "generate", "101"]) + + start_bitcoind() diff --git a/test/test_qt_integration.py b/test/test_qt_integration.py new file mode 100644 index 00000000..b35d4670 --- /dev/null +++ b/test/test_qt_integration.py @@ -0,0 +1,147 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''test of bitcoin-qt integration''' + +import subprocess +import signal +from commontest import local_command, make_wallets +from conftest import start_bitcoind, stop_bitcoind +import os +import pytest +import time +from math import floor + +from joinmarket import Taker, load_program_config, IRCMessageChannel +from joinmarket import validate_address, jm_single +from joinmarket import random_nick, get_p2pk_vbyte +from joinmarket import get_log, choose_sweep_orders, choose_orders, \ + pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object + +import joinmarket.irc +import sendpayment +import bitcoin as btc + +#for running bots as subprocesses +python_cmd = 'python2' +yg_cmd = 'yield-generator-basic.py' +qt_cj_cmd = 'bitcoinqt-coinjoiner.py' + +log = get_log() + +#quite similar to test_regtest.py +def test_qt_integration(setup_qt_int): + makercount = 3 + amount = 1 + wallets = make_wallets(makercount, + wallet_structures=[[1, 0, 0, 0, 0]] * makercount, + mean_amt=10) + ''' + ## bitcoin core's cpu miner sends the coinbase to an obsolete p2pk address + ## so it was believed we'd need to send coins to a p2pkh before using + balance = jm_single().bc_interface.rpc('getbalance', []) + log.debug('total balance = ' + str(balance)) + out_address = jm_single().bc_interface.rpc('getnewaddress', []) + send_chunk = 1000.0 + + if balance < send_chunk: + jm_single().bc_interface.rpc('sendtoaddress', [out_address, balance-amount*2]) + else: + ##sending a tx with too many inputs gives a Transaction Too Large error + send_count = int(floor(balance / send_chunk)) + leftover = balance - send_count*send_chunk + for i in range(send_count): + jm_single().bc_interface.rpc('sendtoaddress', [out_address, send_chunk]) + jm_single().bc_interface.rpc('sendtoaddress', [out_address, leftover-amount*2]) + ''' + + log.debug('stopping bitcoind') + stop_bitcoind() + log.debug('starting bitcoind with no wallet broadcast') + start_bitcoind(['-walletbroadcast=0']) + + jm_procs = [] + for i in range(makercount): + ygp = local_command([python_cmd, yg_cmd, + str(wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + jm_procs.append(ygp) + + log.debug('sleeping for some time to wait for ygens to sync') + time.sleep(240) ##would be improved by detecting when ygens are done + + qt_cjer_proc = local_command([python_cmd, qt_cj_cmd, '--yes', '-N', '2', + '0.1'], bg=True) + jm_procs.append(qt_cjer_proc) + time.sleep(5) + + if btc.secp_present: + destaddr = btc.privkey_to_address( + os.urandom(32), + from_hex=False, + magicbyte=get_p2pk_vbyte()) + else: + destaddr = btc.privkey_to_address( + os.urandom(32), + magicbyte=get_p2pk_vbyte()) + + addr_valid, errormsg = validate_address(destaddr) + assert addr_valid, "Invalid destination address: " + destaddr + \ + ", error message: " + errormsg + #print 'destaddr = ' + destaddr + + txid = jm_single().bc_interface.rpc('sendtoaddress', [destaddr, amount]) + assert txid != None, "something went wrong, txid = None" + + log.debug('sleeping for some time to allow coinjoining to happen') + time.sleep(60) + for p in jm_procs: + #NB *GENTLE* shutdown is essential for + #test coverage reporting! + p.send_signal(signal.SIGINT) + p.wait() + #wait for block generation + + #this is complicated by the fact that the coinjoin address + #in bitcoinqt-coinjoiner.py cant be in the bitcoinqt wallet + blockcount = jm_single().bc_interface.rpc('getblockcount', []) + earlier_blockhash = jm_single().bc_interface.rpc('getblockhash', [blockcount - 3]) + tx_list_json = jm_single().bc_interface.rpc('listsinceblock', [earlier_blockhash]) + checked_txids = set() + found_outputs = [] + import pprint + for tx in tx_list_json['transactions']: + if 'category' not in tx or tx['category'] != 'send': + continue + if tx['txid'] in checked_txids: + continue + checked_txids.add(tx['txid']) + if tx['confirmations'] < 1: + continue + txhex = jm_single().bc_interface.rpc('gettransaction', [tx['txid']]) + if 'hex' not in txhex: + continue + txhex = str(txhex['hex']) + txd = btc.deserialize(txhex) + outputs = dict([(btc.script_to_address(o['script'], + get_p2pk_vbyte()), o['value']) for o in txd['outs']]) + addrs = outputs.keys() + if destaddr in addrs: + #print 'found destaddr' + found_outputs.append(outputs) + + #''' + log.debug('restarting bitcoind back to wallet broadcasting enabled') + stop_bitcoind() + start_bitcoind() + #''' + + log.debug('outputs = ' + str(found_outputs)) + assert len(found_outputs) == 1, " failed to find transaction sending to that address, or found too many txes" + assert found_outputs[0][destaddr] == amount*1e8, " amount not matching" + assert len(found_outputs[0]) > 2, " not a coinjoin" + + return True + +@pytest.fixture(scope="module") +def setup_qt_int(): + load_program_config()