Merge #594: Rewrite revendoring script

2ae7ca9 secp-sys: update README for new vendoring script (Andrew Poelstra) 4b02e9c run new vendor-libsecp.sh; fix upstream CHANGELOG. (Andrew Poelstra) b58a60f rewrite ./vendor-libsecp.sh (Andrew Poelstra) Pull request description: For Nix purposes I need the revendoring script to work without network access and without user interaction. I also realized it would be convenient if the script could figure out what the right version prefix is supposed to be. Then I noticed some shellcheck issues. Anyway I just rewrote the whole thing. I'm now able to run this script within nix and vet that the current contents of the `depend/` directory are consistent with the secp256k1-HEAD-revision.txt, for all commits. ACKs for top commit: tcharding: ACK 2ae7ca9 sanket1729: reACK 2ae7ca9 Tree-SHA512: ea3028e3517b2dbe0f34bcf20685945ecf543fc42e01f10d435432ad290088586b2a2b0f0e94bc3ce59ec38727656eb04eef57c5df6a34da77070e0f288b1d84
rust-bitcoin · Mar 31, 2023 · 493eaf7 · 493eaf7
2 parents 1ad3107 + 2ae7ca9
commit 493eaf7
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 56 deletions.
diff --git a/secp256k1-sys/README.md b/secp256k1-sys/README.md
@@ -20,11 +20,12 @@ This prefix ensures that no symbol collision can happen:
 To update the vendored sources, use the `vendor-libsecp.sh` script:
 
 ```
-$ ./vendor-libsecp.sh depend <version-code> <rev>
+$ ./vendor-libsecp.sh <rev>
 ```
 
-- Where `<version-code>` is the secp256k1-sys version number underscored: `0_1_2`.
-- Where `<rev>` is the git revision of libsecp256k1 to checkout.
+Where `<rev>` is the git revision of libsecp256k1 to checkout. If you do not
+specify a revision, the script will simply clone the repo and use whatever
+revision the default branch is pointing to.
 
 
 ## Linking to external symbols

diff --git a/secp256k1-sys/depend/secp256k1-HEAD-revision.txt b/secp256k1-sys/depend/secp256k1-HEAD-revision.txt
@@ -1,2 +1,2 @@
-# This file was automatically created by ./vendor-libsecp.sh
+# This file was automatically created by vendor-libsecp.sh
 21ffe4b22a9683cf24ae0763359e401d1284cc7a
diff --git a/secp256k1-sys/depend/secp256k1/CHANGELOG.md b/secp256k1-sys/depend/secp256k1/CHANGELOG.md
@@ -7,14 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [0.2.0] - 2022-12-12
 
 ### Added
- - Added `rustsecp256k1_v0_8_0_selftest`, to be used in conjunction with `rustsecp256k1_v0_8_0_context_static`.
+ - Added `secp256k1_selftest`, to be used in conjunction with `secp256k1_context_static`.
 
 ### Changed
  - Enabled modules schnorrsig, extrakeys and ECDH by default in `./configure`.
 
 ### Deprecated
  - Deprecated context flags `SECP256K1_CONTEXT_VERIFY` and `SECP256K1_CONTEXT_SIGN`. Use `SECP256K1_CONTEXT_NONE` instead.
- - Renamed `rustsecp256k1_v0_8_0_context_no_precomp` to `rustsecp256k1_v0_8_0_context_static`.
+ - Renamed `secp256k1_context_no_precomp` to `secp256k1_context_static`.
 
 ### ABI Compatibility
 

diff --git a/secp256k1-sys/vendor-libsecp.sh b/secp256k1-sys/vendor-libsecp.sh
@@ -1,42 +1,96 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
-
-if [ -z "$1" ] | [ -z "$2" ]; then
-  echo "\$1 parameter must be the rust-secp256k1-sys depend directory"
-  echo "\$2 parameter must be the rust-secp256k1-sys version code (M_m_p format)"
-  echo "\$3 parameter (optional) can be the revision to check out"
-  exit 1
+# Set default variables
+if [ -z "$SECP_VENDOR_GIT_ROOT" ]; then
+    SECP_VENDOR_GIT_ROOT="$(git rev-parse --show-toplevel)"
+else
+    SECP_VENDOR_GIT_ROOT="$(realpath "$SECP_VENDOR_GIT_ROOT")"
 fi
+SECP_SYS="$SECP_VENDOR_GIT_ROOT"/secp256k1-sys
+DEFAULT_VERSION_CODE=$(grep version "$SECP_SYS/Cargo.toml" | sed 's/\./_/g' | sed 's/.*"\(.*\)".*/\1/')
+DEFAULT_DEPEND_DIR="$SECP_SYS/depend"
+DEFAULT_SECP_REPO=https://github.com/bitcoin-core/secp256k1.git
+
+: "${SECP_VENDOR_VERSION_CODE:=$DEFAULT_VERSION_CODE}"
+: "${SECP_VENDOR_DEPEND_DIR:=$DEFAULT_DEPEND_DIR}"
+: "${SECP_VENDOR_SECP_REPO:=$DEFAULT_SECP_REPO}"
+# CP_NOT_CLONE lets us just copy a directory rather than git cloning.
+# This is usually a bad idea, since it will bring in build artifacts or any other
+# junk from the source directory, but may be useful during development or CI.
+: "${SECP_VENDOR_CP_NOT_CLONE:=no}"
+
+echo "Using version code $SECP_VENDOR_VERSION_CODE. Set SECP_VENDOR_VERSION_CODE to override."
+echo "Using depend directory $SECP_VENDOR_DEPEND_DIR. Set SECP_VENDOR_DEPEND_DIR to override."
+echo "Using secp repository $SECP_VENDOR_SECP_REPO. Set SECP_VENDOR_SECP_REPO to override."
 
-PARENT_DIR=$1
-VERSIONCODE=$2
-REV=$3
-DIR=secp256k1
-ORIGDIR=$(pwd)
-
-while true; do
-    read -r -p "$PARENT_DIR/$DIR will be deleted [yn]: " yn
-    case $yn in
-        [Yy]* ) break;;
-        [Nn]* ) exit;;
-        * ) echo "Please answer yes or no.";;
+# Parse command-line options
+SECP_REV=""
+FORCE=no
+while (( "$#" )); do
+    case "$1" in
+    -f)
+        FORCE=yes
+        ;;
+    *)
+        if [ -z "$SECP_REV" ]; then
+            echo "Using secp256k1 revision $SECP_REV."
+            SECP_REV="$1"
+        else
+            echo "WARNING: ignoring unknown command-line argument $1"
+        fi
+        ;;
     esac
+    shift
 done
 
-cd "$PARENT_DIR" || exit 1
-rm -rf "$DIR"
-git clone https://github.com/bitcoin-core/secp256k1.git "$DIR"
-cd "$DIR"
-if [ -n "$REV" ]; then
-    git checkout "$REV"
+if [ -z "$SECP_REV" ]; then
+    echo "WARNING: No secp256k1 revision specified. Will use whatever we find at the git repo."
+fi
+echo
+
+# Check if we will do anything destructive.
+
+if [ "$FORCE" == "no" ]; then
+    if ! git diff --quiet -- "*.rs"; then
+        echo "ERROR: There appear to be modified source files. Check these in or pass -f (some source files will be modified to have symbols renamed)."
+        exit 2
+    fi
+    if ! git diff --quiet -- "$SECP_VENDOR_DEPEND_DIR"; then
+        echo "ERROR: The depend directory appears to be modified. Check it in or pass -f (this directory will be deleted)."
+        exit 2
+    fi
 fi
-HEAD=$(git rev-parse HEAD)
-cd ..
-echo "# This file was automatically created by $0" > ./secp256k1-HEAD-revision.txt
-echo "$HEAD" >> ./secp256k1-HEAD-revision.txt
 
-# We need to make some source changes to the files.
+DIR=./secp256k1
+
+pushd "$SECP_VENDOR_DEPEND_DIR" > /dev/null
+rm -rf "$DIR" || true
+
+# Clone the repo. As a special case, if the repo is a local path and we have
+# not specified a revision, just copy the directory rather than using 'git clone'.
+# This lets us use non-git repos or dirty source trees as secp sources.
+if [ "$SECP_VENDOR_CP_NOT_CLONE" == "yes" ]; then
+    cp -r "$SECP_VENDOR_SECP_REPO" "$DIR"
+    chmod -R +w "$DIR" # cp preserves write perms, which if missing will cause patch to fail
+else
+    git clone "$SECP_VENDOR_SECP_REPO" "$DIR"
+fi
+
+# Check out specified revision
+pushd "$DIR" > /dev/null
+if [ -n "$SECP_REV" ]; then
+    git checkout "$SECP_REV"
+fi
+SOURCE_REV=$(git rev-parse HEAD || echo "[unknown revision from $SECP_VENDOR_SECP_REPO]")
+rm -rf .git/ || true
+popd
+
+# Record revision
+echo "# This file was automatically created by $(basename "$0")" > ./secp256k1-HEAD-revision.txt
+echo "$SOURCE_REV" >> ./secp256k1-HEAD-revision.txt
+
+# Patch source files
 
 # To support compiling for WASM, we need to remove all methods that use malloc.
 # To compensate, the secp_context_create and _destroy methods are redefined in Rust.
@@ -46,28 +100,25 @@ patch "$DIR/src/scratch_impl.h" "./scratch_impl.h.patch"
 patch "$DIR/src/util.h" "./util.h.patch"
 
 # Prefix all methods with rustsecp and a version prefix
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i "/^#include/! s/secp256k1_/rustsecp256k1_v${VERSIONCODE}_/g"
-
+find "$DIR" \
+    -not -path '*/\.*' \
+    -not -name "CHANGELOG.md" \
+    -type f \
+    -print0 | xargs -0 sed -i "/^#include/! s/secp256k1_/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_/g"
 # special rule for a method that is not prefixed in libsecp
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i "/^#include/! s/ecdsa_signature_parse_der_lax/rustsecp256k1_v${VERSIONCODE}_ecdsa_signature_parse_der_lax/g"
-
-# TODO: can be removed once 496c5b43b lands in secp-zkp
-find "$DIR" -not -path '*/\.*' -type f -print0 | xargs -0 sed -i 's/^const int CURVE_B/static const int CURVE_B/g'
-
-while true; do
-    read -r -p "Update Rust extern references and Cargo.toml as well? [yn]: " yn
-    case $yn in
-        [Yy]* ) break;;
-        [Nn]* ) exit;;
-        * ) echo "Please answer yes or no.";;
-    esac
-done
-
-cd "$ORIGDIR"
+find "$DIR" \
+    -not -path '*/\.*' \
+    -type f \
+    -print0 | xargs -0 sed -i "/^#include/! s/ecdsa_signature_parse_der_lax/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_ecdsa_signature_parse_der_lax/g"
 
+cd "$SECP_SYS"
 # Update the `links = ` in the manifest file.
-sed -i -r "s/^links = \".*\"$/links = \"rustsecp256k1_v${VERSIONCODE}\"/" Cargo.toml
-
+sed -i -r "s/^links = \".*\"$/links = \"rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}\"/" Cargo.toml
 # Update the extern references in the Rust FFI source files.
-find "./src/" -name "*.rs" -type f -print0 | xargs -0 sed -i -r "s/rustsecp256k1_v[0-9]+_[0-9]+_[0-9]+_(.*)([\"\(])/rustsecp256k1_v${VERSIONCODE}_\1\2/g"
+find "./src/" \
+    -name "*.rs" \
+    -type f \
+    -print0 | xargs -0 sed -i -r "s/rustsecp256k1_v[0-9]+_[0-9]+_[0-9]+_(.*)([\"\(])/rustsecp256k1_v${SECP_VENDOR_VERSION_CODE}_\1\2/g"
+
+popd > /dev/null