From df17eafd3200475c6d756cd1f7bab01c16714ed2 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Mon, 9 Aug 2021 19:17:01 +0200 Subject: [PATCH 1/9] prover_disk: Introduce `nTableBeginPointerCount` --- src/prover_disk.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index 4d1191645..e888148de 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -47,6 +47,7 @@ struct plot_header { // The DiskProver, given a correctly formatted plot file, can efficiently generate valid proofs // of space, for a given challenge. class DiskProver { + const size_t nTableBeginPointerCount{11}; public: // The constructor opens the file, and reads the contents of the file header. The table pointers // will be used to find and seek to all seven tables, at the time of proving. @@ -89,11 +90,11 @@ class DiskProver { memo.resize(Util::TwoBytesToInt(size_buf)); SafeRead(disk_file, memo.data(), memo.size()); - this->table_begin_pointers = std::vector(11, 0); + this->table_begin_pointers = std::vector(nTableBeginPointerCount, 0); this->C2 = std::vector(); uint8_t pointer_buf[8]; - for (uint8_t i = 1; i < 11; i++) { + for (size_t i = 1; i < nTableBeginPointerCount; i++) { SafeRead(disk_file, pointer_buf, 8); this->table_begin_pointers[i] = Util::EightBytesToInt(pointer_buf); } From cbd014bbe3f8d13c0df878724ee7ef2cfd0f7600 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 6 Aug 2021 16:14:38 +0200 Subject: [PATCH 2/9] prover_disk|tests: Implement and test alternative constructor for `DiskProver` --- src/prover_disk.hpp | 30 +++++++++++++++++++++ tests/test.cpp | 63 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index e888148de..9ad03773a 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -118,6 +118,32 @@ class DiskProver { delete[] c2_buf; } + // Note: This constructor presumes the input parameter are valid for the provided file and does + // not validate the file itself. + DiskProver(std::string filename, std::vector memo, std::vector id, uint8_t k, + std::vector table_begin_pointers, std::vector C2) : + filename(std::move(filename)), + memo(std::move(memo)), + id(std::move(id)), + k(k), + table_begin_pointers(std::move(table_begin_pointers)), + C2(std::move(C2)) + { + if (this->id.size() != kIdLen) { + throw std::invalid_argument("Invalid id size: " + std::to_string(this->id.size())); + } + if (k < kMinPlotSize || k > kMaxPlotSize) { + throw std::invalid_argument("Invalid k: " + std::to_string(k)); + } + if (this->table_begin_pointers.size() != nTableBeginPointerCount) { + throw std::invalid_argument("Invalid table_begin_pointers size: " + std::to_string(this->table_begin_pointers.size())); + } + uint32_t nExpectedSize = ((this->table_begin_pointers[10] - this->table_begin_pointers[9]) / (Util::ByteAlign(k) / 8)) - 1; + if (this->C2.size() != nExpectedSize) { + throw std::invalid_argument("Invalid C2 size: " + std::to_string(this->C2.size())); + } + } + ~DiskProver() { std::lock_guard l(_mtx); @@ -130,6 +156,10 @@ class DiskProver { const std::vector& GetMemo() { return memo; } const std::vector& GetId() { return id; } + + const std::vector& GetTableBeginPointers() const noexcept { return table_begin_pointers; } + + const std::vector& GetC2() const noexcept { return C2; } std::string GetFilename() const noexcept { return filename; } diff --git a/tests/test.cpp b/tests/test.cpp index 50d6f6b01..b1c13c8d0 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1026,3 +1026,66 @@ TEST_CASE("FilteredDisk") */ remove("test_file.bin"); } + +TEST_CASE("DiskProver") +{ + SECTION("Construction") + { + std::string filename = "prover_test.plot"; + DiskPlotter plotter = DiskPlotter(); + std::vector memo{1, 2, 3}; + plotter.CreatePlotDisk( + ".", ".", ".", filename, 18, memo.data(), + memo.size(), plot_id_1, 32, 11, 0, + 4000, 2); + DiskProver prover1(filename); + std::string p1_filename = prover1.GetFilename(); + std::vector p1_id = prover1.GetId(); + std::vector p1_memo = prover1.GetMemo(); + uint8_t p1_k = prover1.GetSize(); + std::vector p1_pointers = prover1.GetTableBeginPointers(); + std::vector p1_C2 = prover1.GetC2(); + DiskProver prover2(p1_filename, + p1_memo, + p1_id, + p1_k, + p1_pointers, + p1_C2); + REQUIRE(prover1.GetFilename() == prover2.GetFilename()); + REQUIRE(prover1.GetSize() == prover2.GetSize()); + REQUIRE(prover1.GetId() == prover2.GetId()); + REQUIRE(prover1.GetMemo() == prover2.GetMemo()); + vector hash_input = intToBytes(0, 4); + vector hash(picosha2::k_digest_size); + picosha2::hash256(hash_input.begin(), hash_input.end(), hash.begin(), hash.end()); + vector qualities1 = prover1.GetQualitiesForChallenge(hash.data()); + LargeBits proof1 = prover1.GetFullProof(hash.data(), 0); + vector qualities2 = prover2.GetQualitiesForChallenge(hash.data()); + LargeBits proof2 = prover2.GetFullProof(hash.data(), 0); + REQUIRE(qualities1 == qualities2); + REQUIRE(proof1 == proof2); + // Test "Invalid id" + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, {}, p1_k, p1_pointers, p1_C2)); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, std::vector(kIdLen - 1, 0), p1_k, p1_pointers, p1_C2)); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, std::vector(kIdLen + 1, 0), p1_k, p1_pointers, p1_C2)); + // Test "Invalid k" + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, 0, p1_pointers, p1_C2)); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, kMinPlotSize - 1, p1_pointers, p1_C2)); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, kMaxPlotSize + 1, p1_pointers, p1_C2)); + // Test "Invalid table_begin_pointers size" + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, {}, p1_C2)); + auto invalid_pointers = std::vector(p1_pointers.begin(), p1_pointers.end() - 1); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, invalid_pointers, p1_C2)); + invalid_pointers.push_back(p1_pointers.back()); + invalid_pointers.push_back(p1_pointers.back()); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, invalid_pointers, p1_C2)); + // Test "Invalid C2 size" + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, {}, p1_C2)); + auto invalid_c2 = std::vector(p1_C2.begin(), p1_C2.end() - 1); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, p1_pointers, invalid_c2)); + invalid_c2.push_back(p1_C2.back()); + invalid_c2.push_back(p1_C2.back()); + REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, p1_pointers, invalid_c2)); + REQUIRE(remove(filename.c_str()) == 0); + } +} \ No newline at end of file From 7b9a2c401c5ff464f434beb0f8541cf7708fc069 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 03:30:58 +0200 Subject: [PATCH 3/9] prover_disk|tests: Implement and test move constructor for `DiskProver` --- src/prover_disk.hpp | 13 ++++++++++++- tests/test.cpp | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index 9ad03773a..c4dd79233 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -144,6 +144,17 @@ class DiskProver { } } + DiskProver(DiskProver&& other) noexcept + { + std::lock_guard lock(other._mtx); + filename = std::move(other.filename); + memo = std::move(other.memo); + id = std::move(other.id); + k = other.k; + table_begin_pointers = std::move(other.table_begin_pointers); + C2 = std::move(other.C2); + } + ~DiskProver() { std::lock_guard l(_mtx); @@ -161,7 +172,7 @@ class DiskProver { const std::vector& GetC2() const noexcept { return C2; } - std::string GetFilename() const noexcept { return filename; } + const std::string& GetFilename() const noexcept { return filename; } uint8_t GetSize() const noexcept { return k; } diff --git a/tests/test.cpp b/tests/test.cpp index b1c13c8d0..f7ee53b8f 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1087,5 +1087,24 @@ TEST_CASE("DiskProver") invalid_c2.push_back(p1_C2.back()); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, p1_pointers, invalid_c2)); REQUIRE(remove(filename.c_str()) == 0); + // Test move constructor + auto* p1_filename_ptr = prover1.GetFilename().data(); + auto* p1_memo_ptr = prover1.GetMemo().data(); + auto* p1_id_ptr = prover1.GetId().data(); + auto* p1_table_begin_pointers_ptr = prover1.GetTableBeginPointers().data(); + auto* p1_C2_ptr = prover1.GetC2().data(); + DiskProver prover3(std::move(prover1)); + REQUIRE(prover3.GetFilename().data() == p1_filename_ptr); + REQUIRE(prover3.GetMemo().data() == p1_memo_ptr); + REQUIRE(prover3.GetId().data() == p1_id_ptr); + REQUIRE(prover3.GetTableBeginPointers().data() == p1_table_begin_pointers_ptr); + REQUIRE(prover3.GetC2().data() == p1_C2_ptr); + REQUIRE(prover1.GetFilename().empty()); + REQUIRE(prover1.GetMemo().empty()); + REQUIRE(prover1.GetId().empty()); + REQUIRE(prover1.GetSize() == prover1.GetSize()); + REQUIRE(prover1.GetTableBeginPointers().empty()); + REQUIRE(prover1.GetC2().empty()); + REQUIRE(!prover1.IsValid()); } } \ No newline at end of file From cf0af7966a212022785a1d196fe8004ecf10f917 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 10 Aug 2021 19:52:30 +0200 Subject: [PATCH 4/9] prover_disk|python-bindings|tests: Introduce `Validate`, `IsValid` and a `is_valid` binding. Allows to validate if the prover's file exists and matches expectations. --- python-bindings/chiapos.cpp | 1 + src/prover_disk.hpp | 169 +++++++++++++++++++++------------- tests/test.cpp | 20 ++++ tests/test_python_bindings.py | 1 + 4 files changed, 127 insertions(+), 64 deletions(-) diff --git a/python-bindings/chiapos.cpp b/python-bindings/chiapos.cpp index b887d9902..d3854b942 100644 --- a/python-bindings/chiapos.cpp +++ b/python-bindings/chiapos.cpp @@ -77,6 +77,7 @@ PYBIND11_MODULE(chiapos, m) py::class_(m, "DiskProver") .def(py::init()) + .def("is_valid", [](const DiskProver &dp) { return dp.IsValid(); }) .def( "get_memo", [](DiskProver &dp) { diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index c4dd79233..a184f5b1a 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -53,73 +53,13 @@ class DiskProver { // will be used to find and seek to all seven tables, at the time of proving. explicit DiskProver(const std::string& filename) : id(kIdLen) { - struct plot_header header{}; this->filename = filename; - - std::ifstream disk_file(filename, std::ios::in | std::ios::binary); - - if (!disk_file.is_open()) { - throw std::invalid_argument("Invalid file " + filename); - } - // 19 bytes - "Proof of Space Plot" (utf-8) - // 32 bytes - unique plot id - // 1 byte - k - // 2 bytes - format description length - // x bytes - format description - // 2 bytes - memo length - // x bytes - memo - - SafeRead(disk_file, (uint8_t*)&header, sizeof(header)); - if (memcmp(header.magic, "Proof of Space Plot", sizeof(header.magic)) != 0) - throw std::invalid_argument("Invalid plot header magic"); - - uint16_t fmt_desc_len = Util::TwoBytesToInt(header.fmt_desc_len); - - if (fmt_desc_len == kFormatDescription.size() && - !memcmp(header.fmt_desc, kFormatDescription.c_str(), fmt_desc_len)) { - // OK - } else { - throw std::invalid_argument("Invalid plot file format"); - } - memcpy(id.data(), header.id, sizeof(header.id)); - this->k = header.k; - SafeSeek(disk_file, offsetof(struct plot_header, fmt_desc) + fmt_desc_len); - - uint8_t size_buf[2]; - SafeRead(disk_file, size_buf, 2); - memo.resize(Util::TwoBytesToInt(size_buf)); - SafeRead(disk_file, memo.data(), memo.size()); - - this->table_begin_pointers = std::vector(nTableBeginPointerCount, 0); - this->C2 = std::vector(); - - uint8_t pointer_buf[8]; - for (size_t i = 1; i < nTableBeginPointerCount; i++) { - SafeRead(disk_file, pointer_buf, 8); - this->table_begin_pointers[i] = Util::EightBytesToInt(pointer_buf); - } - - SafeSeek(disk_file, table_begin_pointers[9]); - - uint8_t c2_size = (Util::ByteAlign(k) / 8); - uint32_t c2_entries = (table_begin_pointers[10] - table_begin_pointers[9]) / c2_size; - if (c2_entries == 0 || c2_entries == 1) { - throw std::invalid_argument("Invalid C2 table size"); - } - - // The list of C2 entries is small enough to keep in memory. When proving, we can - // read from disk the C1 and C3 entries. - auto* c2_buf = new uint8_t[c2_size]; - for (uint32_t i = 0; i < c2_entries - 1; i++) { - SafeRead(disk_file, c2_buf, c2_size); - this->C2.push_back(Bits(c2_buf, c2_size, c2_size * 8).Slice(0, k).GetValue()); - } - - delete[] c2_buf; + Read(id, memo, k, table_begin_pointers, C2); } // Note: This constructor presumes the input parameter are valid for the provided file and does - // not validate the file itself. + // not validate the file itself. It's recommended to use `IsValid` after construction to make + // sure the given file matches expectations. DiskProver(std::string filename, std::vector memo, std::vector id, uint8_t k, std::vector table_begin_pointers, std::vector C2) : filename(std::move(filename)), @@ -176,6 +116,39 @@ class DiskProver { uint8_t GetSize() const noexcept { return k; } + void Validate() const + { + uint8_t k_out; + std::vector id_out, memo_out; + std::vector table_begin_pointers_out, C2_out; + Read(id_out, memo_out, k_out, table_begin_pointers_out, C2_out); + if (id != id_out) { + throw std::invalid_argument("EnsureValid: Invalid id"); + } + if (memo != memo_out) { + throw std::invalid_argument("EnsureValid: Invalid memo"); + } + if (k != k_out) { + throw std::invalid_argument("EnsureValid: Invalid k"); + } + if (table_begin_pointers != table_begin_pointers_out) { + throw std::invalid_argument("EnsureValid: Invalid table_begin_pointers"); + } + if (C2 != C2_out) { + throw std::invalid_argument("EnsureValid: Invalid C2"); + } + } + + bool IsValid() const + { + try { + Validate(); + } catch (...) { + return false; + } + return true; + } + // Given a challenge, returns a quality string, which is sha256(challenge + 2 adjecent x // values), from the 64 value proof. Note that this is more efficient than fetching all 64 x // values, which are in different parts of the disk. @@ -280,10 +253,78 @@ class DiskProver { std::string filename; std::vector memo; std::vector id; // Unique plot id - uint8_t k; + uint8_t k{0}; std::vector table_begin_pointers; std::vector C2; + void Read(std::vector& id_in, std::vector& memo_in, uint8_t& k_in, std::vector& table_begin_pointers_in, std::vector& C2_in) const + { + std::lock_guard lock(_mtx); + + std::ifstream disk_file(filename, std::ios::in | std::ios::binary); + + if (!disk_file.is_open()) { + throw std::invalid_argument("Invalid file " + filename); + } + struct plot_header header{}; + // 19 bytes - "Proof of Space Plot" (utf-8) + // 32 bytes - unique plot id + // 1 byte - k + // 2 bytes - format description length + // x bytes - format description + // 2 bytes - memo length + // x bytes - memo + + SafeRead(disk_file, (uint8_t*)&header, sizeof(header)); + if (memcmp(header.magic, "Proof of Space Plot", sizeof(header.magic)) != 0) + throw std::invalid_argument("Invalid plot header magic"); + + uint16_t fmt_desc_len = Util::TwoBytesToInt(header.fmt_desc_len); + + if (fmt_desc_len == kFormatDescription.size() && + !memcmp(header.fmt_desc, kFormatDescription.c_str(), fmt_desc_len)) { + // OK + } else { + throw std::invalid_argument("Invalid plot file format"); + } + id_in = std::vector(kIdLen); + memcpy(id_in.data(), header.id, sizeof(header.id)); + k_in = header.k; + SafeSeek(disk_file, offsetof(struct plot_header, fmt_desc) + fmt_desc_len); + + uint8_t size_buf[2]; + SafeRead(disk_file, size_buf, 2); + memo_in.resize(Util::TwoBytesToInt(size_buf)); + SafeRead(disk_file, memo_in.data(), memo_in.size()); + + table_begin_pointers_in = std::vector(nTableBeginPointerCount, 0); + C2_in = std::vector(); + + uint8_t pointer_buf[8]; + for (size_t i = 1; i < nTableBeginPointerCount; i++) { + SafeRead(disk_file, pointer_buf, 8); + table_begin_pointers_in[i] = Util::EightBytesToInt(pointer_buf); + } + + SafeSeek(disk_file, table_begin_pointers_in[9]); + + uint8_t c2_size = (Util::ByteAlign(k_in) / 8); + uint32_t c2_entries = (table_begin_pointers_in[10] - table_begin_pointers_in[9]) / c2_size; + if (c2_entries == 0 || c2_entries == 1) { + throw std::invalid_argument("Invalid C2 table size"); + } + + // The list of C2 entries is small enough to keep in memory. When proving, we can + // read from disk the C1 and C3 entries. + auto* c2_buf = new uint8_t[c2_size]; + for (uint32_t i = 0; i < c2_entries - 1; i++) { + SafeRead(disk_file, c2_buf, c2_size); + C2_in.push_back(Bits(c2_buf, c2_size, c2_size * 8).Slice(0, k).GetValue()); + } + + delete[] c2_buf; + } + // Using this method instead of simply seeking will prevent segfaults that would arise when // continuing the process of looking up qualities. static void SafeSeek(std::ifstream& disk_file, uint64_t seek_location) { diff --git a/tests/test.cpp b/tests/test.cpp index f7ee53b8f..0475913a4 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1064,14 +1064,28 @@ TEST_CASE("DiskProver") LargeBits proof2 = prover2.GetFullProof(hash.data(), 0); REQUIRE(qualities1 == qualities2); REQUIRE(proof1 == proof2); + + auto test_invalidity = [](std::string filename, std::vector memo, std::vector id, uint8_t k, + std::vector table_begin_pointers, std::vector C2) { + DiskProver prover(filename, memo, id, k, table_begin_pointers, C2); + REQUIRE_THROWS(prover.Validate()); + REQUIRE(!prover.IsValid()); + }; + + // Test "Invalid file" + test_invalidity("invalid", p1_memo, p1_id, p1_k, p1_pointers, p1_C2); // Test "Invalid id" REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, {}, p1_k, p1_pointers, p1_C2)); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, std::vector(kIdLen - 1, 0), p1_k, p1_pointers, p1_C2)); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, std::vector(kIdLen + 1, 0), p1_k, p1_pointers, p1_C2)); + auto invalid_id = p1_id; + invalid_id.back()++; + test_invalidity(p1_filename, p1_memo, invalid_id, p1_k, p1_pointers, p1_C2); // Test "Invalid k" REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, 0, p1_pointers, p1_C2)); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, kMinPlotSize - 1, p1_pointers, p1_C2)); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, kMaxPlotSize + 1, p1_pointers, p1_C2)); + test_invalidity(p1_filename, p1_memo, p1_id, p1_k + 1, p1_pointers, p1_C2); // Test "Invalid table_begin_pointers size" REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, {}, p1_C2)); auto invalid_pointers = std::vector(p1_pointers.begin(), p1_pointers.end() - 1); @@ -1079,6 +1093,9 @@ TEST_CASE("DiskProver") invalid_pointers.push_back(p1_pointers.back()); invalid_pointers.push_back(p1_pointers.back()); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, invalid_pointers, p1_C2)); + invalid_pointers.pop_back(); + invalid_pointers.back()++; + test_invalidity(p1_filename, p1_memo, p1_id, p1_k, invalid_pointers, p1_C2); // Test "Invalid C2 size" REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, {}, p1_C2)); auto invalid_c2 = std::vector(p1_C2.begin(), p1_C2.end() - 1); @@ -1086,6 +1103,9 @@ TEST_CASE("DiskProver") invalid_c2.push_back(p1_C2.back()); invalid_c2.push_back(p1_C2.back()); REQUIRE_THROWS(DiskProver(p1_filename, p1_memo, p1_id, p1_k, p1_pointers, invalid_c2)); + invalid_c2.pop_back(); + invalid_c2.back()++; + test_invalidity(p1_filename, p1_memo, p1_id, p1_k, p1_pointers, invalid_c2); REQUIRE(remove(filename.c_str()) == 0); // Test move constructor auto* p1_filename_ptr = prover1.GetFilename().data(); diff --git a/tests/test_python_bindings.py b/tests/test_python_bindings.py index a04c696be..06cfb0797 100644 --- a/tests/test_python_bindings.py +++ b/tests/test_python_bindings.py @@ -62,6 +62,7 @@ def test_k_21(self): pl = None pr = DiskProver(str(Path("myplot.dat"))) + assert pr.is_valid() total_proofs: int = 0 total_proofs2: int = 0 From 016810c8637c34c477b0fabfb8d42f3721f0ae4d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 03:48:48 +0200 Subject: [PATCH 5/9] python-bindings|prover_disk|tests: Add `pickle` support for `DiskProver` --- python-bindings/chiapos.cpp | 22 ++++++++++++++++++++++ src/prover_disk.hpp | 4 ++-- tests/test_python_bindings.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/python-bindings/chiapos.cpp b/python-bindings/chiapos.cpp index d3854b942..8c0824dd4 100644 --- a/python-bindings/chiapos.cpp +++ b/python-bindings/chiapos.cpp @@ -78,6 +78,28 @@ PYBIND11_MODULE(chiapos, m) py::class_(m, "DiskProver") .def(py::init()) .def("is_valid", [](const DiskProver &dp) { return dp.IsValid(); }) + .def(py::pickle( + [](const DiskProver& dp) { // __getstate__ + return py::make_tuple(dp.GetFilename(), + dp.GetSize(), + dp.GetMemo(), + dp.GetId(), + dp.GetTableBeginPointers(), + dp.GetC2()); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 6) + throw std::runtime_error("Invalid state!"); + + auto filename = t[0].cast(); + auto k = t[1].cast(); + auto memo = t[2].cast>(); + auto id = t[3].cast>(); + auto table_begin_pointers = t[4].cast>(); + auto C2 = t[5].cast>(); + return DiskProver(filename, memo, id, k, + table_begin_pointers, C2); + })) .def( "get_memo", [](DiskProver &dp) { diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index a184f5b1a..61a185f75 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -104,9 +104,9 @@ class DiskProver { Encoding::ANSFree(kC3R); } - const std::vector& GetMemo() { return memo; } + const std::vector& GetMemo() const { return memo; } - const std::vector& GetId() { return id; } + const std::vector& GetId() const { return id; } const std::vector& GetTableBeginPointers() const noexcept { return table_begin_pointers; } diff --git a/tests/test_python_bindings.py b/tests/test_python_bindings.py index 06cfb0797..2b52cf197 100644 --- a/tests/test_python_bindings.py +++ b/tests/test_python_bindings.py @@ -1,4 +1,6 @@ import unittest +import pickle + from chiapos import DiskProver, DiskPlotter, Verifier from hashlib import sha256 from pathlib import Path @@ -177,6 +179,36 @@ def test_faulty_plot_doesnt_crash(self): print(f"Successes: {successes}") print(f"Failures: {failures}") + def test_pickle_support(self): + if not Path("test_plot.dat").exists(): + plot_id: bytes = bytes([i for i in range(0, 32)]) + pl = DiskPlotter() + pl.create_plot_disk( + ".", + ".", + ".", + "test_plot.dat", + 21, + bytes([1, 2, 3, 4, 5]), + plot_id, + 300, + 32, + 8192, + 8, + False, + ) + + prover1: DiskProver = DiskProver(str(Path("test_plot.dat"))) + data = pickle.dumps(prover1) + prover_recovered: DiskProver = pickle.loads(data) + assert prover1.get_size() == prover_recovered.get_size() + assert prover1.get_filename() == prover_recovered.get_filename() + assert prover1.get_id() == prover_recovered.get_id() + assert prover1.get_memo() == prover_recovered.get_memo() + Path("test_plot.dat").unlink() + prover_recovered_invalid: DiskProver = pickle.loads(data) + assert not prover_recovered_invalid.is_valid() + if __name__ == "__main__": unittest.main() From 96f0445bf299421a119d799ab0960d37c83bb529 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 13:13:51 +0200 Subject: [PATCH 6/9] prover_disk: Use snake case for some variables And drop hungarian notation. --- src/prover_disk.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index 61a185f75..c21ae9967 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -47,7 +47,7 @@ struct plot_header { // The DiskProver, given a correctly formatted plot file, can efficiently generate valid proofs // of space, for a given challenge. class DiskProver { - const size_t nTableBeginPointerCount{11}; + const size_t table_begin_pointers_size{11}; public: // The constructor opens the file, and reads the contents of the file header. The table pointers // will be used to find and seek to all seven tables, at the time of proving. @@ -75,11 +75,11 @@ class DiskProver { if (k < kMinPlotSize || k > kMaxPlotSize) { throw std::invalid_argument("Invalid k: " + std::to_string(k)); } - if (this->table_begin_pointers.size() != nTableBeginPointerCount) { + if (this->table_begin_pointers.size() != table_begin_pointers_size) { throw std::invalid_argument("Invalid table_begin_pointers size: " + std::to_string(this->table_begin_pointers.size())); } - uint32_t nExpectedSize = ((this->table_begin_pointers[10] - this->table_begin_pointers[9]) / (Util::ByteAlign(k) / 8)) - 1; - if (this->C2.size() != nExpectedSize) { + uint32_t expected_size = ((this->table_begin_pointers[10] - this->table_begin_pointers[9]) / (Util::ByteAlign(k) / 8)) - 1; + if (this->C2.size() != expected_size) { throw std::invalid_argument("Invalid C2 size: " + std::to_string(this->C2.size())); } } @@ -297,11 +297,11 @@ class DiskProver { memo_in.resize(Util::TwoBytesToInt(size_buf)); SafeRead(disk_file, memo_in.data(), memo_in.size()); - table_begin_pointers_in = std::vector(nTableBeginPointerCount, 0); + table_begin_pointers_in = std::vector(table_begin_pointers_size, 0); C2_in = std::vector(); uint8_t pointer_buf[8]; - for (size_t i = 1; i < nTableBeginPointerCount; i++) { + for (size_t i = 1; i < table_begin_pointers_size; i++) { SafeRead(disk_file, pointer_buf, 8); table_begin_pointers_in[i] = Util::EightBytesToInt(pointer_buf); } From aa66d3a7a6b6c282350c13d831d71a547dcf8c3d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 13:28:09 +0200 Subject: [PATCH 7/9] prover_disk: Make `table_begin_pointers_size` static --- src/prover_disk.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index c21ae9967..280c15c3e 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -47,7 +47,7 @@ struct plot_header { // The DiskProver, given a correctly formatted plot file, can efficiently generate valid proofs // of space, for a given challenge. class DiskProver { - const size_t table_begin_pointers_size{11}; + static const size_t table_begin_pointers_size{11}; public: // The constructor opens the file, and reads the contents of the file header. The table pointers // will be used to find and seek to all seven tables, at the time of proving. From 94cad1af9eeebe6d2b5a0e80be3421b1aa6fd3ed Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 13:28:38 +0200 Subject: [PATCH 8/9] prover_disk: Make `expected_size` const --- src/prover_disk.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index 280c15c3e..e9ef605a7 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -78,7 +78,7 @@ class DiskProver { if (this->table_begin_pointers.size() != table_begin_pointers_size) { throw std::invalid_argument("Invalid table_begin_pointers size: " + std::to_string(this->table_begin_pointers.size())); } - uint32_t expected_size = ((this->table_begin_pointers[10] - this->table_begin_pointers[9]) / (Util::ByteAlign(k) / 8)) - 1; + const uint32_t expected_size = ((this->table_begin_pointers[10] - this->table_begin_pointers[9]) / (Util::ByteAlign(k) / 8)) - 1; if (this->C2.size() != expected_size) { throw std::invalid_argument("Invalid C2 size: " + std::to_string(this->C2.size())); } From a9ca08363e61ac45ed9c3ccae7840464818a7344 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Tue, 24 Aug 2021 13:30:57 +0200 Subject: [PATCH 9/9] prover_disk: Rename `*_in` to `*_out` for `DiskProver::Read` parameters --- src/prover_disk.hpp | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/prover_disk.hpp b/src/prover_disk.hpp index e9ef605a7..4ae638b9f 100644 --- a/src/prover_disk.hpp +++ b/src/prover_disk.hpp @@ -257,7 +257,11 @@ class DiskProver { std::vector table_begin_pointers; std::vector C2; - void Read(std::vector& id_in, std::vector& memo_in, uint8_t& k_in, std::vector& table_begin_pointers_in, std::vector& C2_in) const + void Read(std::vector& id_out, + std::vector& memo_out, + uint8_t& k_out, + std::vector& table_begin_pointers_out, + std::vector& C2_out) const { std::lock_guard lock(_mtx); @@ -287,29 +291,29 @@ class DiskProver { } else { throw std::invalid_argument("Invalid plot file format"); } - id_in = std::vector(kIdLen); - memcpy(id_in.data(), header.id, sizeof(header.id)); - k_in = header.k; + id_out = std::vector(kIdLen); + memcpy(id_out.data(), header.id, sizeof(header.id)); + k_out = header.k; SafeSeek(disk_file, offsetof(struct plot_header, fmt_desc) + fmt_desc_len); uint8_t size_buf[2]; SafeRead(disk_file, size_buf, 2); - memo_in.resize(Util::TwoBytesToInt(size_buf)); - SafeRead(disk_file, memo_in.data(), memo_in.size()); + memo_out.resize(Util::TwoBytesToInt(size_buf)); + SafeRead(disk_file, memo_out.data(), memo_out.size()); - table_begin_pointers_in = std::vector(table_begin_pointers_size, 0); - C2_in = std::vector(); + table_begin_pointers_out = std::vector(table_begin_pointers_size, 0); + C2_out = std::vector(); uint8_t pointer_buf[8]; for (size_t i = 1; i < table_begin_pointers_size; i++) { SafeRead(disk_file, pointer_buf, 8); - table_begin_pointers_in[i] = Util::EightBytesToInt(pointer_buf); + table_begin_pointers_out[i] = Util::EightBytesToInt(pointer_buf); } - SafeSeek(disk_file, table_begin_pointers_in[9]); + SafeSeek(disk_file, table_begin_pointers_out[9]); - uint8_t c2_size = (Util::ByteAlign(k_in) / 8); - uint32_t c2_entries = (table_begin_pointers_in[10] - table_begin_pointers_in[9]) / c2_size; + uint8_t c2_size = (Util::ByteAlign(k_out) / 8); + uint32_t c2_entries = (table_begin_pointers_out[10] - table_begin_pointers_out[9]) / c2_size; if (c2_entries == 0 || c2_entries == 1) { throw std::invalid_argument("Invalid C2 table size"); } @@ -319,7 +323,7 @@ class DiskProver { auto* c2_buf = new uint8_t[c2_size]; for (uint32_t i = 0; i < c2_entries - 1; i++) { SafeRead(disk_file, c2_buf, c2_size); - C2_in.push_back(Bits(c2_buf, c2_size, c2_size * 8).Slice(0, k).GetValue()); + C2_out.push_back(Bits(c2_buf, c2_size, c2_size * 8).Slice(0, k).GetValue()); } delete[] c2_buf;