Proving draft

.github/workflows/build-test-cplusplus.yml

@@ -14,16 +14,21 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Set Env
+        uses: Chia-Network/actions/setjobenv@main
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: cmake, RunTests, and valgrind on ubuntu-20.04
         run: |
           sudo apt update
-          sudo apt-get install valgrind -y
+          sudo apt-get install valgrind libnuma-dev -y
           mkdir build
           cd build
           cmake ../
-          cmake --build . -- -j 6
-          ctest -j 6 --output-on-failure
-          valgrind --leak-check=full --show-leak-kinds=all --errors-for-leak-kinds=all ctest -j 6 --output-on-failure
+          cmake --build . -- -j "$(nproc)"
+          ctest -j "$(nproc)" --output-on-failure
+          valgrind --leak-check=full --show-leak-kinds=all --errors-for-leak-kinds=all ctest -j "$(nproc)" --output-on-failure
 
   asan:
     name: ASAN ubuntu
@@ -32,8 +37,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Set Env
+        uses: Chia-Network/actions/setjobenv@main
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: cmake, RunTests with address- and undefined sanitizer on Ubuntu
         run: |
+          sudo apt-get install libnuma-dev -y
           sudo fallocate -l 16G /swapfile
           sudo chmod 600 /swapfile
           sudo mkswap /swapfile
@@ -42,7 +53,7 @@ jobs:
           mkdir build-asan
           cd build-asan
           cmake -DCMAKE_BUILD_TYPE=ASAN ../
-          cmake --build . -- -j 6
+          cmake --build . -- -j "$(nproc)"
           swapon -s
           ./RunTests
 
@@ -53,25 +64,36 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Set Env
+        uses: Chia-Network/actions/setjobenv@main
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: cmake, RunTests with thread sanitizer on Ubuntu
         run: |
+          sudo apt-get install libnuma-dev -y
           mkdir build-tsan
           cd build-tsan
           cmake -DCMAKE_BUILD_TYPE=TSAN ../
-          cmake --build . -- -j 6
+          cmake --build . -- -j "$(nproc)"
           TSAN_OPTIONS="memory_limit_mb=6000" ./RunTests
 
   windows:
     name: Windows Latest
     runs-on: windows-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set Env
+        uses: Chia-Network/actions/setjobenv@main
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: cmake, RunTests with Windows
-      run: |
-        mkdir build-win
-        cd build-win
-        cmake ..
-        cmake --build . --config Release -j 6
-        ctest -C Release -j 6
+      - name: cmake, RunTests with Windows
+        run: |
+          mkdir build-win
+          cd build-win
+          cmake ..
+          cmake --build . --config Release -j "$(nproc)"
+          ctest -C Release -j "$(nproc)"

.github/workflows/build-wheels.yml

@@ -4,8 +4,8 @@ on:
   push:
     branches:
     - main
-    tags:
-    - '**'
+  release:
+    types: [published]
   pull_request:
     branches:
     - '**'
@@ -19,6 +19,7 @@ jobs:
   build-wheels:
     name: Wheel - ${{ matrix.os.name }} ${{ matrix.python.major-dot-minor }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }}
+    continue-on-error: true
     strategy:
       fail-fast: false
       matrix:
@@ -112,6 +113,12 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Set Env
+      if: env.RUNNER_ARCH != 'ARM64'
+      uses: Chia-Network/actions/setjobenv@main
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     - uses: Chia-Network/actions/setup-python@main
       with:
         python-version: ${{ matrix.python.major-dot-minor }}
@@ -122,6 +129,7 @@ jobs:
 
     - name: Build and test
       env:
+        CI: 1
         CIBW_PRERELEASE_PYTHONS: True
         CIBW_BUILD_VERBOSITY_MACOS: 0
         CIBW_BUILD_VERBOSITY_LINUX: 0
@@ -138,6 +146,7 @@ jobs:
           && which cmake
           && cmake --version
           && uname -a
+          && yum install -y numactl-devel glibc-devel epel-release
         CIBW_BEFORE_BUILD_LINUX: >
           python -m pip install --upgrade pip
         CIBW_ARCHS_MACOS: ${{ matrix.os.cibw-archs-macos[matrix.arch.matrix] }}
@@ -274,6 +283,11 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Set Env
+      uses: Chia-Network/actions/setjobenv@main
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     - uses: Chia-Network/actions/setup-python@main
       with:
         python-version: ${{ matrix.python.major-dot-minor }}
@@ -298,15 +312,15 @@ jobs:
       run: pip install twine
 
     - name: Publish distribution to PyPI
-      if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
+      if: env.RELEASE == 'true' && steps.check_secrets.outputs.HAS_SECRET
       env:
         TWINE_USERNAME: __token__
         TWINE_NON_INTERACTIVE: 1
         TWINE_PASSWORD: ${{ secrets.pypi_password }}
       run: twine upload --non-interactive --skip-existing --verbose 'dist/*'
 
     - name: Publish distribution to Test PyPI
-      if: steps.check_secrets.outputs.HAS_SECRET
+      if: env.PRE_RELEASE == 'true' && steps.check_secrets.outputs.HAS_SECRET
       env:
         TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
         TWINE_USERNAME: __token__

.github/workflows/plot-k27-no-bitfield.yaml

@@ -26,10 +26,11 @@ jobs:
 
     - name: cmake, Plot k=27
       run: |
+        sudo apt-get install libnuma-dev -y
         mkdir build
         cd build
         cmake ../
-        cmake --build . -- -j 6
+        cmake --build . -- -j $(nproc)
         mkdir plots
         mkdir plots/temp
         mkdir plots/final

.github/workflows/plot-k27.yaml

@@ -26,10 +26,11 @@ jobs:
 
     - name: cmake, Plot k=27
       run: |
+        sudo apt-get install libnuma-dev -y
         mkdir build
         cd build
         cmake ../
-        cmake --build . -- -j 6
+        cmake --build . -- -j $(nproc)
         mkdir plots
         mkdir plots/temp
         mkdir plots/final

.gitignore

@@ -25,3 +25,4 @@ build
 .mypy_cache
 *.whl
 venv
+build-tsan

CMakeLists.txt

@@ -34,6 +34,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(cxxopts)
 
+FetchContent_Declare(
+  bladebit
+  GIT_REPOSITORY https://github.com/Chia-Network/bladebit.git
+  GIT_TAG 73b099185f0c473c4a4bb7ae7317adaa43e509de
+)
+set(BLADEBIT_ENABLE_TESTS "off" CACHE STRING "")
+FetchContent_MakeAvailable(bladebit)
+
+
 FetchContent_Declare(
   gulrak
   GIT_REPOSITORY https://github.com/gulrak/filesystem.git
@@ -57,6 +66,7 @@ include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/../lib/FiniteStateEntropy/lib
   ${CMAKE_CURRENT_SOURCE_DIR}/src
   ${CMAKE_CURRENT_SOURCE_DIR}/test
+  ${bladebit_SOURCE_DIR}/src
   )
 
 add_library(fse ${FSE_FILES})
@@ -165,7 +175,8 @@ target_link_libraries(RunTests
   PRIVATE
     fse
     Threads::Threads
-    Catch2::Catch2
+    Catch2::Catch2WithMain
+    lib_bladebit
 )
 
 find_package(Threads REQUIRED)
@@ -175,26 +186,26 @@ target_include_directories(uint128 PUBLIC uint128_t)
 
 target_compile_features(fse PUBLIC cxx_std_17)
 target_compile_features(chiapos PUBLIC cxx_std_17)
-target_compile_features(RunTests PUBLIC cxx_std_17)
+# target_compile_features(RunTests PUBLIC cxx_std_17)
 
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-  target_link_libraries(chiapos PRIVATE fse Threads::Threads)
-  target_link_libraries(ProofOfSpace fse Threads::Threads)
+  target_link_libraries(chiapos PRIVATE fse Threads::Threads lib_bladebit)
+  target_link_libraries(ProofOfSpace fse Threads::Threads lib_bladebit)
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD")
-  target_link_libraries(chiapos PRIVATE fse Threads::Threads)
-  target_link_libraries(ProofOfSpace fse Threads::Threads)
+  target_link_libraries(chiapos PRIVATE fse Threads::Threads lib_bladebit)
+  target_link_libraries(ProofOfSpace fse Threads::Threads lib_bladebit)
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
-  target_link_libraries(chiapos PRIVATE fse Threads::Threads)
-  target_link_libraries(ProofOfSpace fse Threads::Threads)
+  target_link_libraries(chiapos PRIVATE fse Threads::Threads lib_bladebit)
+  target_link_libraries(ProofOfSpace fse Threads::Threads lib_bladebit)
 elseif (MSVC)
-  target_link_libraries(chiapos PRIVATE fse Threads::Threads uint128)
-  target_link_libraries(ProofOfSpace fse Threads::Threads uint128)
+  target_link_libraries(chiapos PRIVATE fse Threads::Threads uint128 lib_bladebit)
+  target_link_libraries(ProofOfSpace fse Threads::Threads uint128 lib_bladebit)
   target_link_libraries(RunTests PRIVATE uint128)
 else()
-  target_link_libraries(chiapos PRIVATE fse stdc++fs Threads::Threads)
-  target_link_libraries(ProofOfSpace fse stdc++fs Threads::Threads)
+  target_link_libraries(chiapos PRIVATE fse stdc++fs Threads::Threads lib_bladebit)
+  target_link_libraries(ProofOfSpace fse stdc++fs Threads::Threads lib_bladebit)
   target_link_libraries(RunTests PRIVATE stdc++fs)
 endif()
 
-enable_testing()
-add_test(NAME RunTests COMMAND RunTests)
+#enable_testing()
+#add_test(NAME RunTests COMMAND RunTests)

python-bindings/chiapos.cpp

@@ -105,6 +105,7 @@ PYBIND11_MODULE(chiapos, m)
                 return py::bytes(reinterpret_cast<const char*>(id.data()), id.size());
             })
         .def("get_size", [](DiskProver &dp) { return dp.GetSize(); })
+        .def("get_compresion_level", [](DiskProver &dp) { return dp.GetCompressionLevel(); })
         .def("get_filename", [](DiskProver &dp) { return dp.GetFilename(); })
         .def(
             "get_qualities_for_challenge",
@@ -179,6 +180,10 @@ PYBIND11_MODULE(chiapos, m)
                 delete[] quality_buf;
                 return stdx::optional<py::bytes>(quality_py);
             });
+
+    py::class_<ContextQueue>(m, "ContextQueue")
+        .def("init", &ContextQueue::init);
+    m.attr("decompresser_context_queue") = &decompresser_context_queue;
 }
 
 #endif  // PYTHON_BINDINGS_PYTHON_BINDINGS_HPP_