HSM 3: PKCS#11 walking skeleton (#727)

Initial PKCS#11 signer support (#689)

* Adds new `Pkcs11Signer` and `MockSigner` signers.

* Temporarily extends `KrillSigner` with an `alternate_config` mode to avoid failing to initialize/login to the same PKCS#11 library twice in one process, until configuration based selection of signers is supported.

* Improves the `SignerInfo` aggregate store to:
  * Store the `PublicKey` directly thus avoiding unecessary (de)serialization.
  * Avoid unnecessary and confusing complexity by using generated UUIDs for signer handles instead of  `<key_identifier>-<internal_private_key_id>`.
  
* Extracts common backend "probing" functionality out of the KMIP signer to a generic testable dedicated module.

* Improves the `SignerRouter` with:
  * Additional trace level logging.
  * A set of tests using the new `MockSigner`.

* Standardizes key deletion error handling across signers.

* Consolidates signer implementations in `<signer>/signer.rs` rather than `internal.rs`.

* Extends the GitHub Actions CI test to:
  * Also test Krill in PKCS#11 mode against SoftHSMv2.
  * Always dump HSM logs at the end of the test.
  * Replaces the `hsm-tests` Rust feature with separate `hsm-tests-kmip` and `hsm-tests-pkcs11` Rust features.
  * Test with the same Rust versions as the normal CI build.

* Fixes compilation failure under Rust 1.47.0 (which also bumps the kmip-dependency to v0.4.1).

* Renames confusing error codes:
  * SignerUnavailable -> TemporarilyUnavailable
  * SignerUnusable -> PermanentlyUnusable

.github/workflows/ci.yml

@@ -68,20 +68,21 @@ jobs:
         path: target/ui/
         if-no-files-found: ignore
 
-  hsmtest:
-    name: hsmtest
+  pykmip-test:
+    name: pykmip-test
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        features: ["hsm", "hsm,hsm-tests"]
+        rust: [1.47.0, stable, beta]
+        features: ["hsm", "hsm,hsm-tests-kmip"]
     steps:
     - name: Checkout repository
       uses: actions/checkout@v1
 
     - name: Install Rust
       uses: hecrj/setup-rust-action@v1
       with:
-        rust-version: stable
+        rust-version: ${{ matrix.rust }}
 
     - uses: actions/setup-python@v2
       with:
@@ -103,10 +104,47 @@ jobs:
         sleep 5s
         openssl s_client -connect 127.0.0.1:5696 || true
         cd -
-        cargo test --no-default-features --features hsm,hsm-tests -- --test-threads=1 2>&1
+        cargo test --no-default-features --features ${{ matrix.features }} -- --test-threads=1 2>&1
 
-    - name: Dump PyKMIP log
+    - name: Dump the PyKMIP log
+      if: always()
       working-directory: test-resources/pykmip
       run: |
         ls -la
-        cat server.log
+        cat server.log
+
+  softhsm2-test:
+    name: softhsm2-test
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        rust: [1.47.0, stable, beta]
+        features: ["hsm,hsm-tests-pkcs11"]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v1
+
+    - name: Install Rust
+      uses: hecrj/setup-rust-action@v1
+      with:
+        rust-version: ${{ matrix.rust }}
+
+    - name: Install SoftHSM2
+      run: |
+        sudo apt update
+        sudo apt install -y softhsm2
+        sudo usermod -aG softhsm $(whoami)
+        sg softhsm -c "softhsm2-util --init-token --slot 0 --label \"My token 1\" --pin 1234 --so-pin 1234"
+
+    - name: Compile the tests
+      run: |
+        cargo build --tests --no-default-features --features ${{ matrix.features }}
+
+    - name: Run the tests against SoftHSM2
+      run: |
+        sg softhsm -c "cargo test --no-default-features --features ${{ matrix.features }} -- --test-threads=1 2>&1"
+
+    - name: Dump the SoftHSM2 log
+      if: always()
+      run: |
+        cat /var/log/syslog

Cargo.toml

@@ -36,12 +36,14 @@ hex                   = "^0.4"
 hyper                 = { version = "^0.14", features = ["server"] }
 intervaltree          = "0.2.6"
 jmespatch             = { version = "^0.3", features = ["sync"], optional = true }
-kmip                  = { version = "0.4.0", package = "kmip-protocol", features = ["tls-with-openssl"], optional = true }
+kmip                  = { version = "0.4.1", package = "kmip-protocol", features = ["tls-with-openssl"], optional = true }
 libflate              = "^1"
 log                   = "^0.4"
+once_cell             = { version = "^1.7.2", optional = true }
 openidconnect         = { version = "^2.0.0", optional = true, default_features = false }
 openssl               = { version = "^0.10", features = ["v110"] }
 oso                   = { version = "^0.12", optional = true, default_features = false }
+pkcs11                = { version = "^0.5", optional = true }
 r2d2                  = { version = "0.8.9", optional = true }
 rand                  = "^0.8"
 regex                 = { version = "^1.4", optional = true, default_features = false, features = ["std"] }
@@ -60,6 +62,13 @@ urlparse              = { version = "^0.7", optional = true }
 uuid                  = { version = "^0.8", features = [ "v4"] }
 xml-rs                = "^0.8"
 
+# Disable compiler optimizations for the pkcs11 crate because otherwise with a release build the
+# `pReserved = ptr::null_mut()` assignment done by `CK_C_INITIALIZE_ARGS::default()` appears to be optimized out. This
+# causes SoftHSMv2 to fail with error CKR_ARGUMENTS_BAD and to log to syslog "SoftHSM.cpp(436): pReserved must be set to
+# NULL_PTR". Disabling optimizations for the pkcs11 crate "solves" this problem.
+[profile.release.package.pkcs11]
+opt-level = 0
+
 [target.'cfg(unix)'.dependencies]
 libc            = "^0.2"
 syslog          = "^4.0"
@@ -76,12 +85,13 @@ static-openssl = [ "openssl/vendored" ]
 # Preview features - not ready for production use
 aspa = []
 rta = []
-hsm = ["backoff", "kmip", "r2d2"]
+hsm = ["backoff", "kmip", "once_cell", "pkcs11", "r2d2"]
 
 # Internal features - not for external use
 all-except-ui-tests = [ "multi-user", "rta", "static-openssl", "aspa" ]
 ui-tests = []
-hsm-tests = []
+hsm-tests-kmip = []
+hsm-tests-pkcs11 = []
 
 # Make sure that Krill crashes on panics, rather than losing threads and
 # limping on in a bad state.