import sys
import shelve
import fileinput
import bsddb
# import ipdb
# from IPython.core.debugger import Tracer

from math import sqrt
from time import sleep
import datetime
from calendar import timegm

sys.path.append('.')
sys.path.append('bitcointools')
sys.path.append('BitcoinArmory')
print sys.path

from armoryengine import *
from armoryengine.BDM import BDM_BLOCKCHAIN_READY, TheBDM
from bitcointools.deserialize import extract_public_key
from bitcointools.util import short_hex, long_hex

# Run with:
# PYTHONPATH=$PYTHONPATH:/home//bitcoin/armory/; python armory_block_reader.py --satoshi-datadir ~/.bitcoin_backup

def get_block_datetime(timestamp):
# block_datetime = datetime.utcfromtimestamp(timestamp)
block_datetime = datetime.utcfromtimestamp(timestamp)
return "%d-%02d-%02d-%02d-%02d-%02d"%(block_datetime.year, block_datetime.month,
block_datetime.day, block_datetime.hour,
block_datetime.minute, block_datetime.second)

class BlockReader:
def __init__(self):
self.index_db = shelve.open('armory_index');
self.bdm = TheBDM
self.verbose = False
self.coinbase = '0000000000000000000000000000000000000000000000000000000000000000'
# self.pk_dict = {}
self.pk_dict = bsddb.hashopen('pk_dict.bdb', 'n', cachesize=2000000)

def cursory_check(self, t, lo=None, hi=None):
if lo is None or hi is None:
return False

loblk = self.bdm.getHeaderByHeight(lo)
lopblk = PyBlockHeader().fromCpp(loblk)

hiblk = self.bdm.getHeaderByHeight(hi)
hipblk = PyBlockHeader().fromCpp(hiblk)

if self.verbose:
print 'Block: %i, @ %s' % (lo, get_block_datetime(lopblk.timestamp))
print 'Block: %i, @ %s' % (hi, get_block_datetime(hipblk.timestamp))

return lopblk.timestamp < t and hipblk.timestamp > t

def binary_search(self, t, lo, hi):
pos = int((lo + hi) / 2)
blk = self.bdm.getHeaderByHeight(pos)
pblk = PyBlockHeader().fromCpp(blk)

# Write to db
if not self.index_db.has_key(str(pos)):
self.index_db[str(pos)] = get_block_datetime(pblk.timestamp)
if self.verbose:
print 'Lo: %i, Hi: %i, Pos: %i(%s)' % (lo, hi, pos, get_block_datetime(pblk.timestamp))

if (hi - lo) < 2:
return self.binary_search(t, lo-1, lo+1)
if (hi - lo) == 2:
return pos
if pblk.timestamp < t:
return self.binary_search(t, pos, hi)
elif pblk.timestamp > t:
return self.binary_search(t, lo, pos)

def load_block_chain(self):
start = datetime.datetime.now()
self.bdm.setBlocking(True)
self.bdm.setOnlineMode(True)

# The setOnlineMode should block until blockchain loading is complete
print 'Loading blockchain took %0.1f sec' % (datetime.datetime.now() - start)

# Indexing
print 'Indexing ...'
if not self.index_db.has_key('top_block_height'):
self.index_db['top_block_height'] = self.bdm.getTopBlockHeight()

print 'Top Block Height: ', self.index_db['top_block_height']
# t1 = timegm(datetime(2011,01,01,0,0,0).timetuple())
# t2 = timegm(datetime(2011,01,02,0,0,0).timetuple())

# silkroad-arrestg
t1 = timegm(datetime(2013,3,25,18,0,0).timetuple())
t2 = timegm(datetime(2013,10,25,18,0,0).timetuple())

# # silkroad-arrest
# t1 = timegm(datetime(2013,10,23,0,0,0).timetuple())
# t2 = timegm(datetime(2013,10,26,0,0,0).timetuple())

print '------------------------------------------'
t1_check = self.cursory_check(t1, lo=0, hi=self.bdm.getTopBlockHeight())
t2_check = self.cursory_check(t2, lo=0, hi=self.bdm.getTopBlockHeight())
print 'Cursory Check t1 %s: %s' % (get_block_datetime(t1), t1_check)
print 'Cursory Check t2 %s: %s' % (get_block_datetime(t2), t2_check)

print '------------------------------------------'
t1_pos = self.binary_search(t1, lo=0, hi=self.bdm.getTopBlockHeight())
# t1_pos = 0;
t2_pos = self.binary_search(t2, lo=0, hi=self.bdm.getTopBlockHeight())
print 'Binary Search t1 %s: %s' % (get_block_datetime(t1), t1_pos)
print 'Binary Search t2 %s: %s' % (get_block_datetime(t2), t2_pos)

print '- Range [%i, %i] --------------------------' % (t1_pos, t2_pos+1)
return range(t1_pos, t2_pos+1)

# pblk = PyBlockHeader().fromCpp(TheBDM.getTopBlockHeader())
# block_datetime = datetime.utcfromtimestamp(pblk.timestamp)
# dt = "%d-%02d-%02d-%02d-%02d-%02d"%(block_datetime.year, block_datetime.month,
# block_datetime.day, block_datetime.hour,
# block_datetime.minute, block_datetime.second)
# print 'Block: %s, @ %s\n' % (pblk.pprint(), dt)

# print '-PBLK-------------------------------------'
# print pblk.pprint()
# print '------------------------------------------'
# print '-BLK--------------------------------------'
# print blk.pprint()
# print '------------------------------------------'

# topBlock = TheBDM.getTopBlockHeight()

# print '\n\nCurrent Top Block is:', topBlock
# TheBDM.getTopBlockHeader().pprint()

# PyBlock
# ==============
# version =
# hashPrev = Prev Hash
# hashMerkleRoot = MerkleRoot
# nTime = Timestamp
# nBits
# nNonce = Nonce
# transactions = getTxRefPtrList()

# PyTx
# ==============
# hash = TxHash
# version = Version
# lockTime = LockTime
# = nInputs
# = nOutputs

# PyTxIn
# ==============
# prevout_hash <=> PrevTxHash
# prevout_n <=> TxOutIndex
# scriptSig <=> Script
# sequence <=> Seq

# PyTxOut
# value <=> Value
# scriptPubKey <= Script: PubKey() OP_CHECKSIG??
def ptx_print(self, tx, dt, writeout=False):
# out = 'TImeslot dt: ', dt
out = ''
# out = {}
ptx = PyTx().fromCpp(tx)
tx_hash = ptx.getHashHex(endianness=BIGENDIAN)
n_inputs = len(ptx.inputs)
n_outputs = len(ptx.outputs)
lock_time = ptx.lockTime

# Get TxIn info
for i in range(tx.getNumTxIn()):
ptxin = PyTxIn().fromCpp(tx.getTxIn(i))
# print '==> TxIn:', i
# print ptxin.pprint()
prev_tx_hash = binary_to_hex(ptxin.outpoint.txHash, BIGENDIAN)

# Only if writing
if not writeout: continue

if prev_tx_hash == self.coinbase:
out = ''.join([out, 'in\t' + tx_hash + '\tcoinbase\t' + dt + '\n'])
else:
inAddr160 = TxInScriptExtractAddr160IfAvail(ptxin)

pk = '(None)'
if len(inAddr160)>0:
pk = hash160_to_addrStr(inAddr160)

# if pk == '(None)' and prev_tx_hash != self.coinbase:
# print '========================= '
# print '(None) ==== ', inAddr160
# print 'tx_hash ', tx_hash
# print 'ptx_hash ', prev_tx_hash
# print '========================= '
# # print ptx.pprint()
# # print '========================= '
# # print ptxin.pprint()
# sys.exit(1)

out = ''.join([out, 'in\t' + tx_hash + '\t' + prev_tx_hash + '\t' + \
str(ptxin.outpoint.txOutIndex) + '\t' + \
pk + '\t' + dt + '\n'])
# print out

index = 0;
for i in range(tx.getNumTxOut()):
ptxout = PyTxOut().fromCpp(tx.getTxOut(i))
# print '==> TxOut:', i
# print ptxout.pprint()
recip = hash160_to_addrStr(tx.getTxOut(i).getRecipientAddr());
pk = TxOutScriptExtractAddrStr(ptxout.binScript)
if writeout:
out = ''.join([out, 'out\t' + tx_hash + '\t' + str(index) + '\t' + pk + \
'\t' + str(float(ptxout.value)/1.0e8) + '\t' + dt + '\n'])
index += 1

# if prev_tx_hash == self.coinbase:
# self.pk_dict[str(tx_hash)] = str(pk)
# print '========================= '
# print 'Prev Tx Hash: ', prev_tx_hash
# print 'Tx Hash: ', tx_hash
# print 'pk: ', pk, tx.getNumTxOut()
# print '========================= '

return out

def load_block(self, blockj, writeout=False):
# Block info
blk = self.bdm.getHeaderByHeight(blockj)
pblk = PyBlockHeader().fromCpp(blk)

if self.verbose:
print '-PBLK-------------------------------------'
print pblk.pprint()
print '------------------------------------------'
# print '-BLK--------------------------------------'
# print blk.pprint()
# print '------------------------------------------'
dt = get_block_datetime(pblk.timestamp)

# Block tx list
# Tracer()()
txList = blk.getTxRefPtrList()
# print '===> TxLIst: ', len(txList)

# For each tx in list
out = []
for txref in txList:
tx = txref.getTxCopy()
# print '=========================='
try:
if writeout:
out.append(self.ptx_print(tx, dt, writeout=True))
else:
self.ptx_print(tx, dt, writeout=False)
except:
pass

if self.verbose:
print '=========================='
print out
print '=========================='
# print tx.pprint()
return len(txList), ''.join(out)

if __name__ == "__main__":
reader = BlockReader()
blocks = reader.load_block_chain();

total_tx = 0;
files = ['transactions-0.txt']
f = open(files[-1], 'w')
for idx,block in enumerate(range(min(blocks), max(blocks))):
if idx % 1000 == 0 and idx > 0:
sys.stdout.write('%i blocks (%i transactions), ' % (idx, total_tx))
sys.stdout.flush()

if idx % 10000 == 0 and idx > 0:
f.close()
files.append('transactions-%i.txt' % idx)
f = open(files[-1], 'w')

# Write out only if block in range
writeout = block in blocks
num_tx, info = reader.load_block(block, writeout=writeout)
if writeout:
total_tx += num_tx
f.write(info)
if f is not None: f.close()
print 'Done: %i blocks (%i transactions) ' % (idx, total_tx)

Nikhil18

Posts

Merits