Re: the fastest possible way to mass-generate addresses in Python
I have modified to create 10 million addresses to make the difference more clear.
10 million addresses generated in less than 26 seconds. Rate = 387.071 (Python).
==> this is a performance boost of additional + 32.7 % on my computer. Crazy!
thank you for pointing out @arulbero. Great!
Code:
$ time python3 citb0in_multicore_secrets.py
Quote
real 0m38,349s
user 5m47,453s
sys 0m16,060s
user 5m47,453s
sys 0m16,060s
Code:
$ time python3 citb0in_multicore_secrets_splitsave.py
Quote
real 0m25,835s
user 5m57,795s
sys 0m14,681s
user 5m57,795s
sys 0m14,681s
10 million addresses generated in less than 26 seconds. Rate = 387.071 (Python).
==> this is a performance boost of additional + 32.7 % on my computer. Crazy!
thank you for pointing out @arulbero. Great! You don't use any specific function of numpy, then I suggest you to eliminate numpy
you have to generate a random key k with k < n = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141, not k < 2**256
you can comment the check at this line:
https://github.com/iceland2k14/secp256k1/blob/main/secp256k1.py#L290
This is your code optimized:
Code:
#!/usr/bin/env python3
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import secrets
import secp256k1 as ice
# how many cores to use
num_cores = 10
#num_cores = os.cpu_count()
# Set the number of addresses to generate
num_addresses = 10000000
# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, i):
addr_type = [2] * (end - start)
is_compressed = [True] * (end - start)
n = [0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141] * (end - start)
f = open("addresses_1M_multicore_secrets" + str(i) + ".txt", "w")
# Generate a list of random private keys using "secrets" library
private_keys = list(map(secrets.randbelow,n))
# Use secp256k1 to convert the private keys to addresses
thread_addresses = list(map(ice.privatekey_to_address, addr_type, is_compressed, private_keys))
# Write the addresses in the thread file
list(map(lambda x:f.write(x+"\n"),thread_addresses))
f.close()
return
# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
# Divide the addresses evenly among the available CPU cores
addresses_per_core = num_addresses // num_cores
# Submit a task for each batch of addresses to the executor
tasks = []
for i in range(num_cores):
start = i * addresses_per_core
end = (i+1) * addresses_per_core
tasks.append(executor.submit(worker, start, end, i))
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import secrets
import secp256k1 as ice
# how many cores to use
num_cores = 10
#num_cores = os.cpu_count()
# Set the number of addresses to generate
num_addresses = 10000000
# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, i):
addr_type = [2] * (end - start)
is_compressed = [True] * (end - start)
n = [0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141] * (end - start)
f = open("addresses_1M_multicore_secrets" + str(i) + ".txt", "w")
# Generate a list of random private keys using "secrets" library
private_keys = list(map(secrets.randbelow,n))
# Use secp256k1 to convert the private keys to addresses
thread_addresses = list(map(ice.privatekey_to_address, addr_type, is_compressed, private_keys))
# Write the addresses in the thread file
list(map(lambda x:f.write(x+"\n"),thread_addresses))
f.close()
return
# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
# Divide the addresses evenly among the available CPU cores
addresses_per_core = num_addresses // num_cores
# Submit a task for each batch of addresses to the executor
tasks = []
for i in range(num_cores):
start = i * addresses_per_core
end = (i+1) * addresses_per_core
tasks.append(executor.submit(worker, start, end, i))