msc_utils_parsing.py

#!/usr/bin/python

#######################################################
#                                                     #
#  Copyright Masterchain Grazcoin Grimentz 2013-2014  #
#  https://github.com/grazcoin/mastercoin-tools       #
#  https://masterchain.info                           #
#  masterchain@@bitmessage.ch                         #
#  License AGPLv3                                     #
#                                                     #
#######################################################

from msc_utils_obelisk import *

mint2b_addr='3Mint2B5ECNdXDZJneJ1XtKmrkmnMbwBbN'
mchain_addr='1MchainXySvRuhdAcJHFfyGLY47P3AEyP9'
donate_addr='1DonateVsLU9zwhgcdWcaWNaaz4MnkWMmv'

transaction_type_dict={'0000':'Simple send', '0014':'Sell offer', '0016':'Sell accept'}
sell_offer_action_dict={'00':'Undefined', '01':'New', '02':'Update', '03':'Cancel'}

first_exodus_bootstrap_block=249498
last_exodus_bootstrap_block=255365
exodus_bootstrap_orig_deadline=1377993600
exodus_bootstrap_deadline=1377993874
seconds_in_one_year=31556926
multisig_disabled=False
dust_limit=5555
MAX_PUBKEY_IN_BIP11=3
MAX_COMMAND_TRIES=3
features_enable_dict={'distributed exchange':290630}
LAST_VALIDATED_BLOCK_NUMBER_FILE='last_validated_block.txt'
max_payment_timeframe=255

# get dict of minted currencies
# example: {"GRZ": {"currency_id": 1, "exodus": "1GRazCon4gDqTh1pMNyh1xHVWnbQEVPfW8", "name": "GRZ coin"},..}
currencies_per_symbol_dict=load_dict_from_file('/home/dev/masterchain-mint2b/mastercoin-tools/general/extracted_currencies.json', skip_error=True)


# get coins list (without Bitcoin) and dict of name to $exodus-$currency_id
coins_symbols_list=currencies_per_symbol_dict.keys()


exodus_scan_list=[]
for cs in coins_symbols_list:
    exo=currencies_per_symbol_dict[cs]['exodus']
    if exo != "":
        seen=False
        for e in exodus_scan_list:
            if exo == e:
                seen=True
        if not seen:
            exodus_scan_list.append(exo)

#try:
#    coins_symbols_list.remove('BTC')
#except ValueError:
#    pass
coins_list=[]
coins_dict={}
for s in coins_symbols_list:
    coins_list.append(currencies_per_symbol_dict[s]['name'])
    coins_dict[currencies_per_symbol_dict[s]['name']]=currencies_per_symbol_dict[s]['exodus']+'-'+str(currencies_per_symbol_dict[s]['currency_id'])

# same dict with other keys: exodus and currency_id
# example: {u'1GRazCon4gDqTh1pMNyh1xHVWnbQEVPfW8': {1: {u'currency_id': 1, 'symbol': u'GRZ', u'name': u'GRZ coin'}, 2: {u'currency_id': 2, 'symbol': u'TGRZ', u'name': u'Test GRZ coin'}},..}
# example: {u'GRZ coin': {u'currency_id': 1, 'symbol': u'GRZ', u'exodus': u'1GRazCon4gDqTh1pMNyh1xHVWnbQEVPfW8', u'name': u'GRZ coin'},...}
currencies_per_exodus_dict={}
currencies_per_name_dict={}

for s in coins_symbols_list:
    # run over all currecy symbols to get exoduses
    currency_dict=currencies_per_symbol_dict[s]
    currency_dict['symbol']=s
    exo=currency_dict['exodus']
    # init dicts if necessary
    if not currencies_per_exodus_dict.has_key(exo):
        currencies_per_exodus_dict[exo]={}
    # update dict
    exodus_currency_dict=currency_dict
    currencies_per_exodus_dict[exo][exodus_currency_dict['currency_id']]=exodus_currency_dict

    name=currency_dict['name']
    # init dicts if necessary
    if not currencies_per_name_dict.has_key(name):
        currencies_per_name_dict[name]={}
    # update dict
    name_currency_dict=currency_dict
    currencies_per_name_dict[name]=exodus_currency_dict

def get_donation_addresses():
    # FIXME: check 1Donate chain
    # get all recipients with 0.00000001 DNT
    donate_addr_dict=load_dict_from_file('addr/'+donate_addr+'.json', all_list=True, skip_error=True)
    tx_list=donate_addr_dict[donate_addr+'-1']['sent_transactions']
    donation_addresses_list=[]
    for tx in tx_list:
        donation_addresses_list.append(tx['to_address'])
    return donation_addresses_list

# used as a key function for sorting outputs of msc tx
def get_dataSequenceNum(item):
    try:
        data_script=item['script'].split()[3].zfill(42)
        dataSequenceNum=data_script[2:4]
        return dataSequenceNum
    except KeyError, IndexError:
        return None

def get_currency_name_from_dict(currencyId, chain='1EXoDusjGwvnjZUyKkxZ4UHEf77z6A5S4P'):
    if not currencies_per_exodus_dict.has_key(chain):
        return 'Unknown currency chain '+str(chain)
    if currencies_per_exodus_dict[chain].has_key(int(currencyId,16)):
        return currencies_per_exodus_dict[chain][int(currencyId,16)]['name']
    else:
        return 'Unknown currency id '+str(currencyId)+' on chain '+chain

def get_currency_symbol_from_name(name):
    try:
        return currencies_per_name_dict[name]['symbol']
    except KeyError:
        return ''

def get_transaction_type_from_dict(transactionType):
    if transaction_type_dict.has_key(transactionType):
        return transaction_type_dict[transactionType]
    else:
        return 'Unknown transaction type '+str(transactionType)

def extract_name(addr):
    addr=addr.strip()
    # verify a valid bitcoin address
    if not is_valid_bitcoin_address(addr):
        return (False, 'invalid adderss')

    # is there already such currency?
    if currencies_per_exodus_dict.has_key(addr):
        d=currencies_per_exodus_dict[addr]
        info(addr+' already extracted as '+d[1]['symbol'])
        return (True, d[1]['symbol'])

    # run on address, skip numbers and vowels, and uppercase
    # don't allow first T (heading T is for test coins)
    full_name=re.sub('[0-9,a,e,i,o,u,A,E,I,O,U]', '', addr)
    if full_name[0].lower()=='t' or len(full_name)<3:
        return (False, 'cannot start with T')
    # start from length 3 and on
    for l in range(3,len(full_name)):
        name=full_name[:l].upper()
        name_exists=currencies_per_symbol_dict.has_key(name)
        if not name_exists:
            return (True, name)
    return(False, 'no free name for address')

def bootstrap_dict_per_tx(block, tx_hash, address, value, dacoins):
    tx_dict={"block": str(block), "tx_hash": tx_hash, "currency_str": "Mastercoin and Test Mastecoin", "to_address": address, "from_address": "exodus", "exodus": True, "tx_method_str": "exodus", "orig_value":value ,"formatted_amount": from_satoshi(dacoins), "tx_type_str": "exodus"}
    return tx_dict

def parse_data_script(data_script):
    parse_dict={}
    if len(data_script)<42:
        info('invalid data script '+data_script.encode('hex_codec'))
        return parse_dict
    parse_dict['baseCoin']=data_script[0:2] # 00 for normal bitcoin (different for multisig?)
    parse_dict['dataSequenceNum']=data_script[2:4]
    parse_dict['transactionVersion']=data_script[4:8]
    parse_dict['transactionType']=data_script[8:12]
    parse_dict['currencyId']=data_script[12:20]
    parse_dict['amount']=data_script[20:36]
    parse_dict['bitcoin_amount_desired']=data_script[36:52]
    parse_dict['block_time_limit']=data_script[52:54]
    return parse_dict

def parse_2nd_data_script(data_script):
    parse_dict={}
    if len(data_script)<42:
        info('invalid data script '+data_script.encode('hex_codec'))
        return parse_dict
    parse_dict['fee_required']=data_script[4:10]
    parse_dict['action']=data_script[10:12]
    parse_dict['should_be_zeros']=data_script[12:54]
    try:
        parse_dict['action_str']=sell_offer_action_dict[parse_dict['action']]
    except KeyError:
        parse_dict['action_str']='unknown '+str(parse_dict['action'])
    return parse_dict

def parse_mint(tx, tx_hash='unknown'):
    json_tx=get_json_tx(tx)
    outputs_list=json_tx['outputs']
    
    from_address=''
    to_address=''
    total_inputs=0
    total_outputs=0
    try:
        inputs=json_tx['inputs']
        for i in inputs:
            if i['address'] != None:
                if from_address != '':
                    from_address+=';'
                from_address+=i['address']
            else:
                from_address='not signed'
            input_value=get_value_from_output(i['previous_output'])
            if input_value==None:
                error('failed get_value_from_output')
            total_inputs+=input_value
    except KeyError, IndexError:
        error('inputs error')
    try:
        for o in outputs_list:
            if o['address'] != None:
                if to_address != '':
                    to_address+=';'
                to_address+=o['address']+':'+from_satoshi((o['value']))
            total_outputs+=(o['value'])
    except KeyError, IndexError:
        error('outputs error')

    # FIXME allow donations addresses list
    mint2b_outputs=0.0;
    donation_outputs=0.0;
    for recipient_and_amount in to_address.split(';'):
        (recipient,amount)=recipient_and_amount.split(':')
        if recipient==mint2b_addr:
            mint2b_outputs+=float(amount)
        for a in get_donation_addresses():
            debug('checking '+a)
            if recipient==a:
                info('donation to '+a+' of '+str(amount))
                donation_outputs+=float(amount)

    if mint2b_outputs < 0.01:
        # ignore minting with less than minimal payment
        info('invalid mint transaction (mint with less than minimal payment) at tx '+tx_hash)
        return {'invalid':(True,'mint with less than minimal payment'), 'tx_hash':tx_hash}

    currency_tuple=extract_name(from_address)
    if currency_tuple[0]==True:
        currency_symbol=currency_tuple[1]
    else:
        currency_symbol='unknown'

    # divide satoshis by 1000 to get total currency units
    # any donation is considered as 50% theoretical payment
    theoretical_and_actual_payment=mint2b_outputs+0.5*donation_outputs
    new_minted_coins=(to_satoshi(theoretical_and_actual_payment)+0.0)/1000

    # A fair price for running the service and maintainig it for you at least 5 years
    # would be 5 BTC.
    # Let's assume market cap of new coin is 100 BTC:
    # if your payment (practical + theoretical by donations) is less than 5 BTC, the protocol says
    # that the rest is paid using the new minted currency
    # For the case that you paid 1BTC and made no donations - 4 BTC are missing which is 4%

    if theoretical_and_actual_payment < 5.0:
        percent_for_masterchain=5.0-theoretical_and_actual_payment
    else:
        percent_for_masterchain=0.0

    formatted_owner_amount=formatted_decimal(new_minted_coins*(100.0-percent_for_masterchain)/100.0)
    formatted_payment_amount=formatted_decimal(new_minted_coins*percent_for_masterchain/100.0)

    parse_dict={}
    # mint tx is from to to from :)
    parse_dict['from_address']=mint2b_addr # included in to_address
    parse_dict['exodus_scan']=from_address
    parse_dict['to_address']=from_address
    parse_dict['fee']=from_satoshi(total_inputs-total_outputs)
    parse_dict['tx_hash']=tx_hash
    parse_dict['currency_str']=currency_symbol+' coin'
    parse_dict['formatted_amount']=formatted_owner_amount
    parse_dict['formatted_payment']=formatted_payment_amount
    parse_dict['formatted_donation']=formatted_decimal(donation_outputs)
    parse_dict['outputs']=to_address
    parse_dict['icon']='exodus'
    parse_dict['icon_text']='Mint currency'
    parse_dict['tx_type_str']='mint'
    parse_dict['color']='bgc-done'
    return parse_dict

def parse_bitcoin_payment(tx, tx_hash='unknown'):
    json_tx=get_json_tx(tx)
    outputs_list=json_tx['outputs']
    
    from_address=''
    to_address=''
    total_inputs=0
    total_outputs=0
    try:
        inputs=json_tx['inputs']
        for i in inputs:
            if i['address'] != None:
                if from_address != '':
                    from_address+=';'
                from_address+=i['address']
            else:
                from_address='not signed'
            input_value=get_value_from_output(i['previous_output'])
            if input_value==None:
                error('failed get_value_from_output')
            total_inputs+=input_value
    except KeyError, IndexError:
        error('inputs error')
    try:
        for o in outputs_list:
            if o['address'] != None:
                if to_address != '':
                    to_address+=';'
                to_address+=o['address']+':'+from_satoshi((o['value']))
            total_outputs+=(o['value'])
    except KeyError, IndexError:
        error('outputs error')

    parse_dict={}
    parse_dict['from_address']=from_address
    parse_dict['to_address']=to_address
    parse_dict['fee']=from_satoshi(total_inputs-total_outputs)
    parse_dict['tx_hash']=tx_hash
    parse_dict['invalid']=(True,'bitcoin payment')
    parse_dict['icon']='bitcoin'
    parse_dict['icon_text']='Bitcoin payment'
    parse_dict['color']='bgc-done'
    return parse_dict

def peek_and_decode(outputs_list):
    # locate data output by checking:
    # Bytes two to eight must equal 00
    # Byte nine must equal 01 or 02
    data_output=None
    data_seq=None
    for o in outputs_list:
        data_script=o['script'].split()[3].zfill(42)
        data_dict=parse_data_script(data_script)
        if (data_dict['transactionType'] == '0000' and \
            ((data_dict['currencyId'] == '00000001' or \
              data_dict['currencyId'] == '00000002'))):
            # invalidate if data was already found before among those outputs
            if data_output != None:
                info('invalid mastercoin tx (multiple valid looking data addresses)')
                return ((True,'multiple valid looking data addresses'), None, None)
            data_output=o
            data_seq=get_dataSequenceNum(o)
    return ((False, ''), data_output, data_seq)

def class_A_Level_1(outputs_list):
    (validity_tuple, data_output, data_seq)=peek_and_decode(outputs_list)
    if (validity_tuple[0]==True):
        return (validity_tuple, None, None)

    if data_output == None:
        info('no data output found')
        return ((True,'no data output found'), None, None)

    recipient=None
    # get the sequence number of this address and add one to it to get the recipient
    recipient_seq=(int(data_seq,16)+1)%256
    for o in outputs_list:
        seq=get_dataSequenceNum(o)
        if int(seq,16)==int(recipient_seq):
            # taking the first one (there may be more)
            recipient=o['address']
    # on failure with 3 outputs case, take non data/exodus to be the recipient
    if len(outputs_list) == 3:
        for o in outputs_list:
            if o['address'] != msc_globals.exodus_scan and o != data_output:
                recipient = o['address']
    return ((False,''), data_output, recipient)

# "Class A" transaction
def parse_simple_basic(tx, tx_hash='unknown', after_bootstrap=True):
    json_tx=get_json_tx(tx)
    outputs_list=json_tx['outputs']
    (outputs_list_no_exodus, outputs_to_exodus, different_outputs_values, invalid)=examine_outputs(outputs_list, tx_hash, tx)
    if invalid != None:
        info(str(invalid[1])+' on '+tx_hash)
        return {'invalid':invalid, 'tx_hash':tx_hash}

    num_of_outputs=len(outputs_list)

    # collect all "from addresses" (normally only a single one)
    from_address=''
    try:
        inputs=json_tx['inputs']
        inputs_values_dict={}
        for i in inputs:
            input_value=get_value_from_output(i['previous_output'])
            if input_value == None:
                error('failed get_value_from_output')
            input_address=i['address']
            if inputs_values_dict.has_key(input_address):
                inputs_values_dict[input_address]+=int(input_value)
            else:
                inputs_values_dict[input_address]=int(input_value)

        # the from address is the one with the highest value
        from_address=max(inputs_values_dict, key=inputs_values_dict.get)

        if from_address == None:
            info('invalid from address (address with largest value is None) at tx '+tx_hash)
            return {'invalid':(True,'address with largest value is None'), 'tx_hash':tx_hash}


        #######################################################################
        # follow Class A P&D https://github.com/mastercoin-MSC/spec/issues/29 #
        #######################################################################
        # Level 1
        # Take all the outputs that have the same value as the value to the Exodus address
        # (the first exodus output is checked here)
        exodus_value=outputs_to_exodus[0]['value']
        outputs_with_exodus_value=different_outputs_values[exodus_value]
        # locate data address by checking:
        # Bytes two to eight must equal 00
        # Byte nine must equal 01 or 02
        (invalidity_tuple, data_output, recipient)=class_A_Level_1(outputs_with_exodus_value)
        # check for invalids
        if invalidity_tuple[0] == True:
            info(invalidity_tuple[1]+' '+tx_hash)
        if data_output == None or recipient == None:
            # Level 2
            # If the sequence number can't be found you can expand the searching range to
            # include all outputs
            (invalidity_tuple, level2_data_output, level2_recipient)=class_A_Level_1(outputs_list)
            # check for invalids
            if invalidity_tuple[0] == True:
                info(invalidity_tuple[1]+' '+tx_hash)
            if level2_data_output != None and level2_recipient != None:
                debug('Level 2 peek and decode for '+tx_hash)
                data_output=level2_data_output
                recipient=level2_recipient
            else:
                # Level 3
                # all output values are equal in size and if there are three outputs
                # of these type of outputs total
                if (len(different_outputs_values)==1 and len(different_outputs_values[0])==3 and data_output != None):
                    debug('Level 3 peek and decode for '+tx_hash)
                    # Collect all outputs and remove the data address and the Exodus output.
                    # The remaining output is the recipient address.
                    all_addresses=[d['address'] for d in different_outputs_values[0]]
                    all_addresses.remove(msc_globals.exodus_scan)
                    all_addresses.remove(data_output['address'])
                    recipient=all_addresses[0]
                else:
                    info('invalid mastercoin tx (failed all peek and decode levels) '+tx_hash)
                    return parse_bitcoin_payment(tx, tx_hash)
        else:
            debug('Level 1 peek and decode for '+tx_hash)         

        to_address=recipient
        data_script=data_output['script'].split()[3].zfill(42)
        data_dict=parse_data_script(data_script)
        if len(data_dict) >= 6: # at least the basic 6 fields were parsed
            parse_dict=data_dict
            parse_dict['tx_hash']=tx_hash
            parse_dict['from_address']=from_address
            parse_dict['to_address']=to_address
            parse_dict['formatted_amount']=from_hex_satoshi(data_dict['amount'])
            parse_dict['currency_str']=get_currency_name_from_dict(data_dict['currencyId'], msc_globals.exodus_scan)
            parse_dict['tx_type_str']=get_transaction_type_from_dict(data_dict['transactionType'])
            parse_dict['tx_method_str']='basic'
            return parse_dict
        else:
            info('invalid mastercoin tx with less than 6 fields '+tx_hash)
            return {'invalid':(True,'invalid mastercoin tx with less than 6 fields'), 'tx_hash':tx_hash}
    except (KeyError, IndexError, TypeError) as e:
        info('invalid mastercoin tx ('+str(e)+') at tx '+tx_hash)
        return {'invalid':(True,'bad parsing'), 'tx_hash':tx_hash}

def get_obfus_str_list(address, length):
       obfus_str_list=[]
       obfus_str_list.append(get_sha256(address)) # 1st obfus is simple sha256
       for i in range(length):
           if i<length-1: # one less obfus str is needed (the first was not counted)
               obfus_str_list.append(get_sha256(obfus_str_list[i].upper())) # i'th obfus is sha256 of upper prev
       return obfus_str_list

def parse_multisig(tx, tx_hash='unknown'):
    if multisig_disabled:
        info('multisig is disabled: '+tx_hash)
        return {}
    parsed_json_tx=get_json_tx(tx)
    parse_dict={}
    input_addr=''
    for i in parsed_json_tx['inputs']:
        previous_output=i['previous_output']
        if input_addr == '':
            input_addr=get_address_from_output(previous_output)
        else:
            if get_address_from_output(previous_output) != input_addr:
                info('Bad multiple inputs on: '+tx_hash)
                return {'tx_hash':tx_hash, 'invalid':(True, 'Bad multiple inputs')}
    all_outputs=parsed_json_tx['outputs']
    (outputs_list_no_exodus, outputs_to_exodus, different_outputs_values, invalid)=examine_outputs(all_outputs, tx_hash, tx)
    if invalid != None:
        info(str(invalid[1])+' on '+tx_hash)
        return {'tx_hash':tx_hash, 'invalid':invalid}
        
    tx_dust=outputs_to_exodus[0]['value']
    dust_outputs=different_outputs_values[tx_dust]
    to_address='unknown'
    for o in dust_outputs: # assume the only other dust is to recipient
        if o['address']!=msc_globals.exodus_scan:
            to_address=o['address']
            continue

    for o in outputs_list_no_exodus:
        if o['address']==None: # This should be the multisig
            script=o['script']
            # verify that it is a multisig
            if not script.endswith('checkmultisig'):
                error('Bad multisig data script '+script)
            fields=script.split('[ ')

            # more sanity checks on BIP11
            max_pubkeys=int(fields[-1].split()[-2])
            req_pubkeys=int(fields[0])
            if req_pubkeys != 1:
                info('error m-of-n with m different than 1 ('+str(req_pubkeys)+'). skipping tx '+tx_hash)
                return {'tx_hash':tx_hash, 'invalid':(True, 'error m-of-n with m different than 1')}
            if max_pubkeys < 2 or max_pubkeys > 3:
                info('error m-of-n with n out of range ('+str(max_pubkeys)+'). skipping tx '+tx_hash)
                return {'tx_hash':tx_hash, 'invalid':(True, 'error m-of-n with n out of range')}

            # parse the BIP11 pubkey list
            data_script_list=[]
            for i in range(MAX_PUBKEY_IN_BIP11-1):
                index=i+2 # the index of the i'th pubkey
                try:
                    data_script_list.append(fields[index].split(' ]')[0])
                except IndexError:
                    break

            # prepare place holder lists for obfus,deobfus,data_dict
            dataHex_deobfuscated_list=[]
            data_dict_list=[]
           
            if input_addr == None:
                info('none input address (BIP11 inputs are not supported yet)')
                return {'tx_hash':tx_hash, 'invalid':(True, 'not supported input (BIP11/BIP16)')}

            list_length=len(data_script_list)
            obfus_str_list=get_obfus_str_list(input_addr, list_length)

            for i in range(list_length):
                dataHex_deobfuscated_list.append(get_string_xor(data_script_list[i][2:-2],obfus_str_list[i][:62]).zfill(64)+'00')

            # deobfuscated list is ready
            #info(dataHex_deobfuscated_list)

            try:
                data_dict=parse_data_script(dataHex_deobfuscated_list[0])
            except IndexError:
                error('cannot parse dataHex_deobfuscated_list')

            # no recipient? allow for sell offer
            if to_address=='unknown' and (data_dict['transactionType'] == '0000' or data_dict['transactionType'] == '0016'):
                info('no recipient tx '+tx_hash)
                return {'tx_hash':tx_hash, 'invalid':(True, 'no recipient')}

            if len(data_dict) >= 6: # at least 6 basic fields got parse on the first dataHex
                amount=int(data_dict['amount'],16)/100000000.0
                parse_dict=data_dict
                parse_dict['tx_hash']=tx_hash
                parse_dict['formatted_amount'] = formatted_decimal(amount)
                parse_dict['currency_str'] = get_currency_name_from_dict(data_dict['currencyId'], msc_globals.exodus_scan)
                parse_dict['tx_type_str'] = get_transaction_type_from_dict(data_dict['transactionType'])
                parse_dict['tx_method_str'] = 'multisig'

                if data_dict['transactionType'] == '0000': # Simple send
                    # remove irrelevant keys
                    parse_dict.pop('bitcoin_amount_desired', None)
                    parse_dict.pop('block_time_limit', None)
                else:
                    if data_dict['transactionType'] == '0014': # Sell offer
                        # check feature is enabled
                        if get_currency_name_from_dict(data_dict['currencyId'], msc_globals.exodus_scan) == 'Mastercoin':
                            (height,index)=get_tx_index(tx_hash)
                            if height == -1:
                                error('failed getting height of '+tx_hash)
                            if int(features_enable_dict['distributed exchange']) > int(height):
                                info('distributed exchange of msc is not yet enabled '+tx_hash)
                                parse_dict['invalid']=(True, 'distributed exchange of msc is not yet enabled')
                                parse_dict['color']='bgc-invalid'
                                parse_dict['icon_text']='Invalid sell offer'
                                parse_dict['from_address']=input_addr
                                parse_dict['to_address']=to_address
                                return parse_dict

                        bitcoin_amount_desired=int(data_dict['bitcoin_amount_desired'],16)/100000000.0
                        if amount > 0:
                            price_per_coin=bitcoin_amount_desired/amount
                        else:
                            price_per_coin=0

                        # duplicate with another name
                        parse_dict['formatted_amount_available'] = parse_dict['formatted_amount']
                        # format fields
                        parse_dict['formatted_bitcoin_amount_desired']= formatted_decimal(bitcoin_amount_desired)
                        parse_dict['formatted_price_per_coin']= formatted_decimal(price_per_coin)
                        parse_dict['formatted_block_time_limit']= str(int(data_dict['block_time_limit'],16))

                        if len(dataHex_deobfuscated_list)>1: # currently true only for Sell offer (?)
                            data_dict2=parse_2nd_data_script(dataHex_deobfuscated_list[1])

                            # verify positive sell offer amount
                            if amount == 0: # this is allowed only on Cancel action
                                if data_dict['transactionVersion']=='0000' or data_dict2['action_str']=='Cancel':
                                    debug('cancel sell offer with zero amount on '+tx_hash)
                                else:
                                    parse_dict['invalid']=(True,'zero sell offer amount')
                            else:
                                if amount < 0:
                                    info('BUG: negative sell offer amount on '+tx_hash)
                                    parse_dict['invalid']=(True,'negative sell offer amount')


                            if data_dict2['should_be_zeros'] == '0000000000000000000000000000000000000000000' or \
                               data_dict2['should_be_zeros'] == '000000000000000000000000000000000000000000':
                                data_dict2.pop('should_be_zeros')
                                for key in data_dict2:
                                    parse_dict[key]=data_dict2[key]
                                parse_dict['formatted_fee_required'] = from_hex_satoshi(data_dict2['fee_required'])
                            else:
                                parse_dict['invalid']=(True,'invalid last data script in BIP11')
                                return parse_dict
                    else:
                        if data_dict['transactionType'] == '0016': # Sell accept
                            # check feature is enabled
                            if get_currency_name_from_dict(data_dict['currencyId'], msc_globals.exodus_scan) == 'Mastercoin' and tx_hash != 'unknown':
                                (height,index)=get_tx_index(tx_hash)
                                if height == -1 or height == 'failed:':
                                    error('failed getting height of '+tx_hash)
                                if int(features_enable_dict['distributed exchange']) > int(height):
                                    info('distributed exchange of msc is not yet enabled '+tx_hash)
                                    parse_dict['invalid']=(True, 'distributed exchange of msc is not yet enabled')
                                    parse_dict['color']='bgc-invalid'
                                    parse_dict['icon_text']='Invalid sell accept'
                                    parse_dict['from_address']=input_addr
                                    parse_dict['to_address']=to_address
                                    return parse_dict

                            # remove irrelevant keys
                            parse_dict.pop('bitcoin_amount_desired', None)
                            parse_dict.pop('block_time_limit', None)
                            # duplicate with another name
                            parse_dict['formatted_amount_requested'] = parse_dict['formatted_amount']
                            # add place holders
                            parse_dict['bitcoin_required'] = 'Not available'
                            parse_dict['sell_offer_txid'] = 'Not available'
                            parse_dict['payment_txid'] = 'Not available'
                            parse_dict['status'] = 'Awaiting payment'
                            # parse as bitcoin payment to get the tx fee
                            bitcoin_dict=parse_bitcoin_payment(tx, tx_hash)
                            parse_dict['formatted_fee']=bitcoin_dict['fee']

                        else: # non valid tx type
                            return {'tx_hash':tx_hash, 'invalid':(True, 'non supported tx type '+data_dict['transactionType'])}

        else: # not the multisig output
            # the output with dust
            parse_dict['to_address']=o['address']

    if parse_dict == {}:
        error('Bad parsing of multisig: '+tx_hash)

    parse_dict['from_address']=input_addr
    parse_dict['to_address']=to_address
                
    return parse_dict

def examine_outputs(outputs_list, tx_hash, raw_tx):
        # if we're here, then 1EXoDus is within the outputs. Remove it, but ...
        outputs_list_no_exodus=[]
        outputs_to_exodus=[]
        different_outputs_values={}
        for o in outputs_list:
            if o['address']!=msc_globals.exodus_scan:
                outputs_list_no_exodus.append(o)
            else:
                outputs_to_exodus.append(o)
            output_value=o['value']
            if different_outputs_values.has_key(output_value):
                different_outputs_values[output_value].append(o)
            else:
                different_outputs_values[output_value]=[o]
        # take care if multiple 1EXoDus exist (for the case that someone sends msc
        # to 1EXoDus, or have 1EXoDus as change address, or sends from 1EXoDus)
        if len(outputs_to_exodus) != 1:
            # support the special case of sending from 1EXoDus
            json_tx=get_json_tx(raw_tx)
            inputs_list=json_tx['inputs']
            from_exodus=False
            for i in inputs_list:
                if i['address']==msc_globals.exodus_scan:
                    from_exodus=True
                    break
            if not from_exodus:
                info("invalid tx with multiple 1EXoDus outputs not from 1EXoDus: "+tx_hash)
                return (None, None, None, (True,'multiple 1EXoDus outputs not from 1EXoDus'))
            else: # 1EXoDus has sent this tx
                # Maximal 3 values are valid (dust, double dust, and change)
                if len(different_outputs_values.keys()) > 3:
                    info("tx sent by exodus with more than 3 different values: "+tx_hash)
                    return (None, None, None, (True,'tx sent by exodus with more than 3 different values'))
                # move the dust exodus from outputs_to_exodus list to the outputs_list_no_exodus one
                if len(different_outputs_values.keys()) == 1: # change is identical to marker
                    debug("tx sent by exodus with single value")
                    # move one item from exodus to no exodus list
                    o=outputs_to_exodus.pop()
                    outputs_list_no_exodus.append(o)
                else:
                    if len(different_outputs_values.keys()) == 2: # change is identical to marker
                        debug("tx sent by exodus with two values")
                        # move one item from exodus to no exodus list
                        o=outputs_to_exodus.pop()
                        outputs_list_no_exodus.append(o)
                    else:
                        debug("tx sent by exodus with 3 values to exodus")
                        # as there is a signle change, dust_value belongs to list with non single item
                        output_values=different_outputs_values.keys()
                        output_values.sort()
                        dust_value=output_values[0]
                        # veify double dust exists
                        if output_values[1] == 2*dust_value:
                            # double dust is [1] so the so the is the change
                            change_value=output_values[2]
                        else:
                            if output_values[2] != 2*dust_value:
                                # double dust is [2] so the so the is the change
                                change_value=output_values[1]
                            else:
                                info('tx by exodus with non valid 3 values to exodus')
                                return (None, None, None, (True,'tx by exodus with non valid 3 values to exodus'))
                  
                        # move the dust item from exodus to no exodus list
                        dust_outputs_to_exodus=[]
                        non_dust_outputs_to_exodus=[]
                        for o in outputs_to_exodus:
                            if o['value']==change_value:
                                dust_outputs_to_exodus.append(o)
                            else:
                                non_dust_outputs_to_exodus.append(o)
                        # move the item
                        outputs_list_no_exodus+=[dust_outputs_to_exodus[0]]
                        outputs_to_exodus=non_dust_outputs_to_exodus+dust_outputs_to_exodus[1:]
        return (outputs_list_no_exodus, outputs_to_exodus, different_outputs_values, None)

def get_tx_method(tx, tx_hash='unknown'): # multisig_simple, multisig, multisig_invalid, basic
        json_tx=get_json_tx(tx)
        outputs_list=json_tx['outputs']
        (outputs_list_no_exodus, outputs_to_exodus, different_outputs_values, invalid)=examine_outputs(outputs_list, hx_hash, tx)

        num_of_outputs=len(outputs_list)

        # check if basic or multisig
        is_basic=True
        for o in outputs_list:
            if is_script_multisig(o['script']):
                if num_of_outputs == 2:
                    return 'multisig_simple'
                else:
                    if num_of_outputs > 2:
                        return 'multisig'
                    else:
                        return 'multisig_invalid'
        # all the rest, which includes exodus and invalids are considered as basic
        return 'basic'