Quote from: kTimesG on Today at 08:37:10 PM

Quote from: farou9 on Today at 08:00:57 PM

No harm in hoping.

I just want to first see how fast can it solve smaller bits 70 down

OK. I cooked up something that uses all cores to build all the points in a given range.

It can do 50 - 60 MK/s, if you have 64 cores. But the problem is the saving - SQLite can only ever write in synchronized mode (which makes sense, since it needs to check the index), and the best I got is around 300.000 inserts per second. Saving to files is a non-sense because then you won't be able to efficiently query the X value (which is the main point after all).

I'll upload the code tomorrow on GitHub so you can test it.

well I have 4 cores so I am technically cooked.
"300k inserts/s" I have a c++ script that gives 30k/s but the way it works is different and will take some days to finish the billion ,it opens 3200 per time until it finishes the hall range storing only the specific prefix opened files .

Code:

#include <iostream>
#include <fstream>
#include <sstream>
#include <iomanip>
#include <string>
#include <vector>
#include <unordered_map>
#include <chrono>
#include <filesystem>
#include <secp256k1.h>

namespace fs = std::filesystem;

std::string to_hex(const unsigned char* data, size_t len) {
    std::ostringstream oss;
    for (size_t i = 0; i < len; ++i)
        oss << std::hex << std::setw(2) << std::setfill('0') << (int)data[i];
    return oss.str();
}

void increment_privkey(unsigned char* privkey) {
    for (int i = 31; i >= 0; --i) {
        if (++privkey[i] != 0) break;
    }
}

int main() {
    const uint64_t num_scalars = 1'000'000'000;
    const size_t batch_size = 3200;
    const std::string output_dir = "x_values_by_prefix";

    fs::create_directories(output_dir); // Create the output directory if it doesn't exist
    secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN);

    for (int batch_start = 0; batch_start < 65536; batch_start += batch_size) {
        std::unordered_map<std::string, std::ofstream> file_map;
        std::vector<std::string> prefixes;

        // Open 3000 files for this batch
        for (int i = batch_start; i < batch_start + batch_size && i < 65536; ++i) {
            std::stringstream ss;
            ss << std::hex << std::setw(4) << std::setfill('0') << i;
            std::string prefix = ss.str();
            prefixes.push_back(prefix);
            
            // Open the file in append mode, to ensure we don't overwrite existing data
            // If the file exists, it appends; if not, it creates a new file.
            file_map[prefix].open(output_dir + "/" + prefix + ".txt", std::ios::app); 
        }

        // Reset private key to scalar = 1
        unsigned char privkey[32] = {0};
        privkey[31] = 1;

        secp256k1_pubkey pubkey;
        unsigned char pubkey_serialized[65];
        size_t pubkey_len = 65;

        size_t stored_count = 0;
        auto start_time = std::chrono::steady_clock::now();

        for (uint64_t i = 1; i <= num_scalars; ++i) {
            if (!secp256k1_ec_pubkey_create(ctx, &pubkey, privkey)) {
                std::cerr << "Invalid private key at scalar " << i << "\n";
                increment_privkey(privkey);
                continue;
            }

            pubkey_len = 65;
            secp256k1_ec_pubkey_serialize(ctx, pubkey_serialized, &pubkey_len, &pubkey, SECP256K1_EC_UNCOMPRESSED);
            std::string x_hex = to_hex(pubkey_serialized + 1, 32);
            std::string prefix = x_hex.substr(0, 4); // Get the first 4 characters of the x-coordinate (prefix)

            // Write to the appropriate file based on the prefix
            if (file_map.find(prefix) != file_map.end()) {
                file_map[prefix] << x_hex << " " << i << "\n";  // Append the x value and scalar index to the file
                ++stored_count;
            }

            increment_privkey(privkey); // Increment the private key for the next iteration

            // Log progress every 100,000 iterations
            if (i % 100000 == 0) {
                auto elapsed = std::chrono::steady_clock::now() - start_time;
                double seconds = std::chrono::duration_cast<std::chrono::seconds>(elapsed).count();
                double rate = i / (seconds ? seconds : 1.0);
                std::cout << "Processed: " << i << ", Stored: " << stored_count
                          << ", Time: " << seconds << "s, Speed: " << rate << " keys/sec\n";
            }
        }

        // Close all files after processing the batch
        for (auto& [_, fout] : file_map) fout.close();

        // Log batch completion
        std::cout << "Finished batch starting from prefix: " << prefixes.front() << " → Stored: " << stored_count << "\n";
    }

    secp256k1_context_destroy(ctx); // Clean up secp256k1 context
    std::cout << "All batches completed!\n";
    return 0;
}