From 58a564d512221707f7ea6410f95ec75107dad1fd Mon Sep 17 00:00:00 2001 From: wz Date: Mon, 12 Aug 2019 14:13:52 +0800 Subject: [PATCH] add merger utxo --- README.md | 5 + toolbar/merger_utxo/README.md | 93 +++++++++++++++++ toolbar/merger_utxo/common/Account.py | 95 +++++++++++++++++ toolbar/merger_utxo/common/Transaction.py | 74 ++++++++++++++ toolbar/merger_utxo/common/UnspentOutputs.py | 29 ++++++ toolbar/merger_utxo/common/__init__.py | 0 toolbar/merger_utxo/common/connection.py | 18 ++++ toolbar/merger_utxo/common/merge_utxo.py | 147 +++++++++++++++++++++++++++ toolbar/merger_utxo/merger_utxo.py | 7 ++ 9 files changed, 468 insertions(+) create mode 100644 toolbar/merger_utxo/README.md create mode 100644 toolbar/merger_utxo/common/Account.py create mode 100644 toolbar/merger_utxo/common/Transaction.py create mode 100644 toolbar/merger_utxo/common/UnspentOutputs.py create mode 100644 toolbar/merger_utxo/common/__init__.py create mode 100644 toolbar/merger_utxo/common/connection.py create mode 100755 toolbar/merger_utxo/common/merge_utxo.py create mode 100755 toolbar/merger_utxo/merger_utxo.py diff --git a/README.md b/README.md index 52870ed3..b9c95e70 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ cast in consensus around, and choose how many rounds of consensus to allocate th [Tool usage details](./cmd/votereward/README.md) + +### Merger utxo + UTXO has been merged to solve the problem that too much UTXO input causes a failed send transaction to fail. + [details](./toolbar/merger_utxo_README.md) + ## License [AGPL v3](./LICENSE) diff --git a/toolbar/merger_utxo/README.md b/toolbar/merger_utxo/README.md new file mode 100644 index 00000000..f68dc3da --- /dev/null +++ b/toolbar/merger_utxo/README.md @@ -0,0 +1,93 @@ +# UTXO merger + + +> **One last disclaimer:** + +**the code we are about to go over is in no way intended to be used as an example of a robust solution.** + +**We wouldn't be responsible for the consequences of using this tool.** + +**please check this python code carefully and use it later.** + + +Requirements: Python 3.x, with requests package + +Dependencies: + ``` + pip install requests + ``` + +Options: + ``` + $ python merger_utxo.py -h +usage: merger_utxo.py [-h] [-o URL] [-a ACCOUNT_ALIAS] [-p PASSWORD] + [-x MAX_AMOUNT] [-s MIN_AMOUNT] [-l] [-m MERGE_LIST] + [-f FOR_LOOP] [-y] + +Vapor merge utxo tool + +optional arguments: + -h, --help show this help message and exit + -o URL, --url URL API url to connect + -a ACCOUNT_ALIAS, --account ACCOUNT_ALIAS + account alias + -p PASSWORD, --pass PASSWORD + account password + -x MAX_AMOUNT, --max MAX_AMOUNT + range lower than max_amount + -s MIN_AMOUNT, --min MIN_AMOUNT + range higher than min_amount + -l, --list Show UTXO list without merge + -m MERGE_LIST, --merge MERGE_LIST + UTXO to merge + -f FOR_LOOP, --forloop FOR_LOOP + size for loop of UTXO to merge + -y, --yes confirm transfer + + ``` + +Example: + ``` +$ python btmspanner.py utxomerger -o http://127.0.0.1:9888 -a your_account_alias -p your_password -x 41250000000 -s 0 -m 20 -f 3 -y + ``` + +Result: +``` +$ python ./merger_utxo.py -o http://127.0.0.1:9889 -a test -p 123456 -x 2000000000 -s 0 -f 1 + 0. 11.00000000 BTM d996363b3443407fe3828517f53551b1dda19b9e7503974892874318d03b9ed3 (mature) + 1. 11.00000000 BTM d9308dae432c592e7f32ba39ddf6c7882350bbd4b0269c5b96842f1df14b31cf (mature) + 2. 11.00000000 BTM c6fd223ffc2475ab0029bc5233be47c289970c141505effe20fe49f06cb575d6 (mature) + 3. 11.00000000 BTM 74851bebb8e94375bd346e36718ce1a4c273ecaff97501ab1985097ff85d7eaf (mature) + 4. 11.00000000 BTM 6610f18f0ecf4238568dea5a3dd1e360c5d3bff74c1bf0fefc19848a0852aabf (mature) + 5. 11.00000000 BTM 4e9eb10481f1c7306ab833f632bc04134979d39605f82b40c4c107015ec15322 (mature) + 6. 11.00000000 BTM 343a7d72cc830b992f8b777d32a78074a6338f642c060905997663a5c4549a8a (mature) + 7. 11.00000000 BTM 338f38f99211c1da076114db3951cfccf57c6612f1ad87bafb4043d161fc9572 (mature) + 8. 11.00000000 BTM 23c3d1210e636a1a30d0769c41ad2fcd25459c5f557d9fbd5b5a414d31f66ca2 (mature) + 9. 11.00000000 BTM 12198e59879b139768b92bd1272fce1a6274afe43829312105b8e36ec76f8c4a (mature) + 10. 11.00000000 BTM 10a420ddb64b34f7d1b3f56230d6c07e0e5c299f98c1ad5fbfe20100736128ab (mature) + 11. 11.00000000 BTM 00055e2354b20a5b88c6c6914ede710b6e19575d24937325461374909446821e (mature) +total size of available utxos is 12 +To merge 12 UTXOs with 132.00000000 BTM totally. + +One last disclaimer: the code we are about to go over is in no way intended to be used as an example of a robust solution. +You will transfer BTM to an address, please check this python code and DO IT later. + +this is the 1 times to merge utxos. -----begin + 0. 11.00000000 BTM d996363b3443407fe3828517f53551b1dda19b9e7503974892874318d03b9ed3 (mature) + 1. 11.00000000 BTM d9308dae432c592e7f32ba39ddf6c7882350bbd4b0269c5b96842f1df14b31cf (mature) + 2. 11.00000000 BTM c6fd223ffc2475ab0029bc5233be47c289970c141505effe20fe49f06cb575d6 (mature) + 3. 11.00000000 BTM 74851bebb8e94375bd346e36718ce1a4c273ecaff97501ab1985097ff85d7eaf (mature) + 4. 11.00000000 BTM 6610f18f0ecf4238568dea5a3dd1e360c5d3bff74c1bf0fefc19848a0852aabf (mature) + 5. 11.00000000 BTM 4e9eb10481f1c7306ab833f632bc04134979d39605f82b40c4c107015ec15322 (mature) + 6. 11.00000000 BTM 343a7d72cc830b992f8b777d32a78074a6338f642c060905997663a5c4549a8a (mature) + 7. 11.00000000 BTM 338f38f99211c1da076114db3951cfccf57c6612f1ad87bafb4043d161fc9572 (mature) + 8. 11.00000000 BTM 23c3d1210e636a1a30d0769c41ad2fcd25459c5f557d9fbd5b5a414d31f66ca2 (mature) + 9. 11.00000000 BTM 12198e59879b139768b92bd1272fce1a6274afe43829312105b8e36ec76f8c4a (mature) + 10. 11.00000000 BTM 10a420ddb64b34f7d1b3f56230d6c07e0e5c299f98c1ad5fbfe20100736128ab (mature) + 11. 11.00000000 BTM 00055e2354b20a5b88c6c6914ede710b6e19575d24937325461374909446821e (mature) +total size of available utxos is 12 +To merge 12 UTXOs with 132.00000000 BTM +Confirm [y/N] y +tx_id: 38b38a6715ef223643a4c961b0f0553edcf6eb67b82546f741e4c391400cffa0 +this is the 1 times to merge utxos. -----end +``` \ No newline at end of file diff --git a/toolbar/merger_utxo/common/Account.py b/toolbar/merger_utxo/common/Account.py new file mode 100644 index 00000000..ffe33513 --- /dev/null +++ b/toolbar/merger_utxo/common/Account.py @@ -0,0 +1,95 @@ +import json + + +class Account(object): + def __init__(self, id, alias, key_index, quorum, xpubs=[], *args, **kwargs): + self.id = id + self.alias = alias + self.key_index = key_index + self.quorum = quorum + self.xpubs = xpubs + + @staticmethod + def list(connection): + response = connection.request("/list-accounts") + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + account_list = list(map(lambda x: Account(**x), resp_json['data'])) + return account_list + elif resp_json['status'] == 'fail': + return resp_json['msg'] + else: + return resp_json + + @staticmethod + def list_address(connection, account_alias, account_id): + + body_json = {"account_alias": account_alias, "account_id": account_id} + + response = connection.request("/list-addresses", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return resp_json['data'], 1 + elif resp_json['status'] == 'fail': + return resp_json['msg'], -1 + else: + return resp_json, 0 + + @staticmethod + def find_by_alias(connection, alias): + account_list = Account.list(connection) + for account in account_list: + if account.alias == alias: + return account + + @staticmethod + def find_address_by_alias(connection, account_alias): + account_id = Account.find_by_alias(connection, account_alias).id + address_list, ret = Account.list_address(connection, account_alias, account_id) + if ret == 1: + return address_list[0]['address'] + + @staticmethod + def create(connection, root_xpubs, alias, quorum): + body_json = {"root_xpubs": root_xpubs, "alias": alias, "quorum": quorum} + response = connection.request("/create-account", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return Account(**resp_json['data']) + elif resp_json['status'] == 'fail': + return resp_json['msg'] + else: + return resp_json + + @staticmethod + def delete(connection, account_info): + # String - account_info, alias or ID of account. + body_json = {"account_info": account_info} + + response = connection.request("/delete-account", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return "true" + else: + return "false" + + @staticmethod + def create_address(connection, account_alias, account_id): + body_json = {"account_alias": account_alias, "account_id": account_id} + + response = connection.request("/create-account-receiver", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == "success": + return resp_json['data'] + else: + return "false" diff --git a/toolbar/merger_utxo/common/Transaction.py b/toolbar/merger_utxo/common/Transaction.py new file mode 100644 index 00000000..9cc2b9ac --- /dev/null +++ b/toolbar/merger_utxo/common/Transaction.py @@ -0,0 +1,74 @@ +import json + +from .Account import Account + + +class Transaction(object): + @staticmethod + def build_transaction(connection, actions): + # ttl: 15min=900000ms + body_json = {"base_transaction": None, "actions": actions, + "ttl": 1, "time_range": 0} + + response = connection.request("/build-transaction", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return resp_json['data'], True + elif resp_json['status'] == 'fail': + return resp_json['msg'], False + else: + return resp_json, False + + @staticmethod + def sign_transaction(connection, password, transaction): + body_json = {"password": password, "transaction": transaction} + response = connection.request("/sign-transaction", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return resp_json['data'], True + elif resp_json['status'] == 'fail': + return resp_json['msg'], False + else: + return resp_json, False + + @staticmethod + def submit_transaction(connection, raw_transaction): + body_json = {"raw_transaction": raw_transaction} + response = connection.request("/submit-transaction", body_json) + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return resp_json['data'] + + +class Action(object): + + @staticmethod + def spend_account(amount, account_id, asset_id): + return { + 'amount': amount, + 'account_id': account_id, + 'asset_id': asset_id, + 'type': 'spend_account' + } + + @staticmethod + def control_address(amount, asset_id, address): + return { + 'amount': amount, + 'asset_id': asset_id, + 'address': address, + 'type': 'control_address' + } + + @staticmethod + def unspent_output(output_id): + return { + 'type': 'spend_account_unspent_output', + 'output_id': output_id + } diff --git a/toolbar/merger_utxo/common/UnspentOutputs.py b/toolbar/merger_utxo/common/UnspentOutputs.py new file mode 100644 index 00000000..ae89a397 --- /dev/null +++ b/toolbar/merger_utxo/common/UnspentOutputs.py @@ -0,0 +1,29 @@ +import json + + +class UnspentOutputs(object): + + @staticmethod + def get_block_height(connection): + response = connection.request("/get-block-count") + + resp_json = json.loads(response.text) + + if resp_json['status'] == 'success': + return resp_json['data']['block_count'], 1 + elif resp_json['status'] == 'fail': + return resp_json['msg'], -1 + else: + return resp_json, 0 + + @staticmethod + def list_UTXO(connection): + response = connection.request("/list-unspent-outputs") + + resp_json = json.loads(response.text) + if resp_json['status'] == 'success': + return resp_json['data'], 1 + elif resp_json['status'] == 'fail': + return resp_json['msg'], -1 + else: + return resp_json, 0 diff --git a/toolbar/merger_utxo/common/__init__.py b/toolbar/merger_utxo/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolbar/merger_utxo/common/connection.py b/toolbar/merger_utxo/common/connection.py new file mode 100644 index 00000000..67091667 --- /dev/null +++ b/toolbar/merger_utxo/common/connection.py @@ -0,0 +1,18 @@ +import requests +import json + + +class Connection(object): + def __init__(self, base_url, token=''): + self.baseUrl = base_url + self.token = token + + def request(self, path, body={}): + url = self.baseUrl + path + headers = {} + resp = requests.post(url, data=json.dumps(body), headers=headers) + return resp + + @staticmethod + def generate(): + return Connection("http://127.0.0.1:9889") diff --git a/toolbar/merger_utxo/common/merge_utxo.py b/toolbar/merger_utxo/common/merge_utxo.py new file mode 100755 index 00000000..490443e1 --- /dev/null +++ b/toolbar/merger_utxo/common/merge_utxo.py @@ -0,0 +1,147 @@ +import argparse +import getpass +import os +import time + +from .Account import Account +from .Transaction import Action, Transaction +from .UnspentOutputs import UnspentOutputs +from .connection import Connection + +parser = argparse.ArgumentParser(description='Vapor merge utxo tool') +parser.add_argument('-o', '--url', default='http://127.0.0.1:9889', dest='url', help='API url to connect') +parser.add_argument('-a', '--account', default=None, dest='account_alias', help='account alias') +parser.add_argument('-p', '--pass', default=None, dest='password', help='account password') +parser.add_argument('-x', '--max', default=41250000000, type=int, dest='max_amount', help='range lower than max_amount') +parser.add_argument('-s', '--min', default=1, type=int, dest='min_amount', help='range higher than min_amount') +parser.add_argument('-l', '--list', action='store_true', dest='only_list', help='Show UTXO list without merge') +parser.add_argument('-m', '--merge', default=90, type=int, dest='merge_list', help='UTXO to merge') +parser.add_argument('-f', '--forloop', default=1, type=int, dest='for_loop', help='size for loop of UTXO to merge') +parser.add_argument('-y', '--yes', action='store_true', default=None, dest='confirm', help='confirm transfer') + + +class VaporException(Exception): + pass + + +class JSONRPCException(Exception): + pass + + +def list_utxo(connection, account_alias, min_amount, max_amount): + mature_utxos = [] + data, ret = UnspentOutputs.list_UTXO(connection=Connection(connection)) + block_height, ret_code = UnspentOutputs.get_block_height(connection=Connection(connection)) + if ret == 1 and ret_code == 1: + for utxo in data: + # append mature utxo to set + if utxo['valid_height'] < block_height and utxo['account_alias'] == account_alias and utxo['asset_alias'] == 'BTM': + mature_utxos.append(utxo) + elif ret == -1: + raise VaporException(data) + + result = [] + for utxo in mature_utxos: + if utxo['amount'] <= max_amount and utxo['amount'] >= min_amount: + result.append(utxo) + + return result + + +def send_tx(connection, utxo_list, to_address, password): + actions = [] + amount = 0 + asset_id = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + + for utxo in utxo_list: + actions.append(Action.unspent_output(output_id=utxo['id'])) + amount += utxo['amount'] + + actions.append(Action.control_address(amount=amount, asset_id=asset_id, address=to_address)) + + time.sleep(1) + transaction,f = Transaction.build_transaction(connection, actions) + if f == False: + return "" + + signed_transaction,f = Transaction.sign_transaction(connection, password, transaction) + if signed_transaction['sign_complete']: + raw_transaction = signed_transaction['transaction']['raw_transaction'] + result = Transaction.submit_transaction(connection, raw_transaction) + return result['tx_id'] + else: + raise VaporException('Sign not complete') + + +def main(): + options = parser.parse_args() + utxo_total = [] + utxolist = list_utxo(options.url, options.account_alias, options.min_amount, options.max_amount) + + for i, utxo in enumerate(utxolist): + print('{:4}. {:13.8f} BTM {}{}'.format(i, utxo['amount'] / 1e8, utxo['id'], ' (mature)')) + if i >= options.merge_list * options.for_loop: + break + utxo_total.append(utxo) + + print("total size of available utxos is {}".format(len(utxolist))) + + if options.only_list: + return + + print('To merge {} UTXOs with {:13.8f} BTM totally.\n'.format(len(utxo_total), + sum(utxo['amount'] for utxo in utxo_total) / 1e8)) + + merge_size = options.merge_list or input('Merge size of UTXOs (5, 13 or 20): ') + for_loop = options.for_loop or input('for loop size (1, 10 or 50): ') + + print( + 'One last disclaimer: the code we are about to go over is in no way intended to be used as an example of a robust solution. ') + print('You will transfer BTM to an address, please check this python code and DO IT later.\n') + + for loops in range(for_loop): + + utxo_mergelist = [] + + # for i in range(merge_size if merge_size <= len(utxolist) else len(utxolist)): + # utxo_mergelist.append(utxolist[i]) + for i in range(loops * merge_size, + ((loops + 1) * merge_size) if ((loops + 1) * merge_size) < len(utxolist) else len(utxolist)): + utxo_mergelist.append(utxolist[i]) + + # print(loops*merge_size, ", ", ((loops+1)*merge_size) if (loops*merge_size) < len(utxolist) else len(utxolist)) + print('this is the {} times to merge utxos. -----begin'.format(loops + 1)) + + for i, utxo in enumerate(utxo_mergelist): + print( + '{:4}. {:13.8f} BTM {}{}'.format(loops * merge_size + i, utxo['amount'] / 1e8, utxo['id'], ' (mature)')) + + print("total size of available utxos is {}".format(len(utxo_mergelist))) + + if len(utxo_mergelist) < 2: + print('Not Merge UTXOs, Exit...') + return + + print('To merge {} UTXOs with {:13.8f} BTM'.format(len(utxo_mergelist), + sum(utxo['amount'] for utxo in utxo_mergelist) / 1e8)) + + if not options.account_alias: + options.account_alias = input('Transfer account alias: ') + + if not options.password: + options.password = getpass.getpass('Vapor Account Password: ') + + if not (options.confirm or input('Confirm [y/N] ').lower() == 'y'): + print('Not Merge UTXOs, Exit...') + return + + to_address = Account.find_address_by_alias(Connection(options.url), options.account_alias) + if not to_address: + to_address = input('Transfer address: ') + + print('tx_id:', send_tx(Connection(options.url), utxo_mergelist, to_address, options.password)) + print('this is the {} times to merge utxos. -----end\n'.format(loops + 1)) + + +if __name__ == '__main__': + main() diff --git a/toolbar/merger_utxo/merger_utxo.py b/toolbar/merger_utxo/merger_utxo.py new file mode 100755 index 00000000..5c35c9c2 --- /dev/null +++ b/toolbar/merger_utxo/merger_utxo.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys +from common import merge_utxo + +if __name__ == "__main__": + sys.exit(merge_utxo.main()) -- 2.11.0