Codegen: option to use BTreeMap as map representation

As of now, protobuf maps are represented with [std::collections::HashMap](https://doc.rust-lang.org/stable/std/collections/struct.HashMap.html), which serves as a robust default. In rarer instances, opting for a different implementation, such as BTreeMap, might be desirable, e.g. for deterministic serialization. This change * adds `btreemaps` codegen option to generate [std::collections::BTreeMap](https://doc.rust-lang.org/std/collections/struct.BTreeMap.html) for maps representation.
stepancheg · Jan 8, 2024 · 615c237 · 615c237
1 parent 4cffee8
commit 615c237
Show file tree

Hide file tree

Showing 27 changed files with 441 additions and 289 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,5 +1,3 @@
-# @generated by ci-gen, do not edit
-
 on:
     - push
     - pull_request
@@ -50,51 +48,6 @@ jobs:
               uses: actions-rs/cargo@v1
               with:
                   command: doc
-    linux-beta-default-features:
-        name: linux beta (default features)
-        runs-on: ubuntu-latest
-        env:
-            RUST_BACKTRACE: 1
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: beta
-                  override: true
-            - name: Cache protobuf
-              uses: actions/cache@v2
-              with:
-                  key: pb-linux-3.19.4
-                  path: ~/pb
-              env:
-                  cache-name: pb
-            - name: Install protobuf
-              run: ci/install-protobuf.sh
-              shell: bash
-              env:
-                  PROTOBUF_VERSION: 3.19.4
-            - name: Protoc check
-              run: protoc --version
-              shell: bash
-            - name: Compile interop
-              run: test-crates/interop/cxx/compile.sh
-              shell: bash
-            - name: Regenerate
-              run: protobuf/regenerate.sh
-              shell: bash
-            - name: Test all
-              run: cargo test --all --all-targets
-              shell: bash
-            - name: Test all
-              run: cargo test
-              shell: bash
-            - name: cargo doc
-              uses: actions-rs/cargo@v1
-              with:
-                  command: doc
     linux-stable-with-bytes:
         name: linux stable (with-bytes)
         runs-on: ubuntu-latest
@@ -137,130 +90,6 @@ jobs:
               with:
                   command: test
                   args: --manifest-path=test-crates/protobuf-codegen-pure-test/Cargo.toml --features=with-bytes
-    linux-nightly-all-features:
-        name: linux nightly (all features)
-        runs-on: ubuntu-latest
-        env:
-            RUST_BACKTRACE: 1
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: nightly
-                  override: true
-            - name: Cache protobuf
-              uses: actions/cache@v2
-              with:
-                  key: pb-linux-3.19.4
-                  path: ~/pb
-              env:
-                  cache-name: pb
-            - name: Install protobuf
-              run: ci/install-protobuf.sh
-              shell: bash
-              env:
-                  PROTOBUF_VERSION: 3.19.4
-            - name: Protoc check
-              run: protoc --version
-              shell: bash
-            - name: Compile interop
-              run: test-crates/interop/cxx/compile.sh
-              shell: bash
-            - name: Test all
-              run: cargo test --all --all-targets --all-features
-              shell: bash
-            - name: Test all
-              run: cargo test --all-features
-              shell: bash
-            - name: cargo doc
-              uses: actions-rs/cargo@v1
-              with:
-                  command: doc
-                  args: --all-features
-    windows-stable-default-features:
-        name: windows stable (default features)
-        runs-on: windows-latest
-        env:
-            RUST_BACKTRACE: 1
-            VCPKGRS_DYNAMIC: 1
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: stable
-                  override: true
-            - name: Cache protobuf
-              uses: actions/cache@v2
-              with:
-                  key: pb-windows-3.19.4
-                  path: ~/pb
-              env:
-                  cache-name: pb
-            - name: Install protobuf
-              run: ci/install-protobuf.sh
-              shell: bash
-              env:
-                  PROTOBUF_VERSION: 3.19.4
-            - name: Protoc check
-              run: protoc --version
-              shell: bash
-            - name: Test all
-              run: cargo test --all --all-targets
-              shell: bash
-    miri-test:
-        name: Miri test
-        runs-on: ubuntu-latest
-        timeout-minutes: 5
-        env:
-            RUST_BACKTRACE: 1
-            RUST_TEST_THREADS: 1
-            MIRIFLAGS: -Zmiri-tag-raw-pointers
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: nightly
-                  override: true
-                  components: miri
-            - name: cargo-miri-setup
-              uses: actions-rs/cargo@v1
-              with:
-                  command: miri
-                  args: setup
-            - name: cargo-miri-test
-              uses: actions-rs/cargo@v1
-              with:
-                  command: miri
-                  args: test -p protobuf --lib --all-features
-    mega-linter:
-        name: mega-linter
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-              with:
-                  fetch-depth: 0
-            - name: mega-linter
-              uses: megalinter/megalinter@v5
-              env:
-                  VALIDATE_ALL_CODEBASE: false
-                  DEFAULT_BRANCH: master
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-                  VALIDATE_JSCPD: false
-                  VALIDATE_PROTOBUF: false
-                  VALIDATE_RUST_CLIPPY: false
-                  VALIDATE_RUST_2015: false
-                  VALIDATE_COPYPASTE: false
-                  VALIDATE_SPELL: false
     rustfmt-check:
         name: rustfmt check
         runs-on: ubuntu-latest
@@ -289,54 +118,3 @@ jobs:
             - name: cargo fmt check
               run: cargo fmt -- --check
               shell: bash
-    cargo-sync-readme:
-        name: Check sync-readme
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: stable
-                  override: true
-            - name: install cargo sync-readme
-              run: cargo install cargo-sync-readme
-              shell: bash
-            - name: sync-readme protobuf
-              run: cd protobuf && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-codegen
-              run: cd protobuf-codegen && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-examples/customize-serde
-              run: cd protobuf-examples/customize-serde && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-examples/vs-prost
-              run: cd protobuf-examples/vs-prost && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-json-mapping
-              run: cd protobuf-json-mapping && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-parse
-              run: cd protobuf-parse && cargo sync-readme --check
-              shell: bash
-            - name: sync-readme protobuf-support
-              run: cd protobuf-support && cargo sync-readme --check
-              shell: bash
-    self-check:
-        name: CI self-check
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: stable
-                  override: true
-            - name: The check
-              run: cargo run -p ci-gen -- --check
-              shell: bash
diff --git a/Cargo.toml b/Cargo.toml
@@ -16,6 +16,7 @@ members = [
     "test-crates/perftest/bytes",
     "test-crates/perftest/misc",
     "test-crates/perftest/vs-cxx",
+    "test-crates/protobuf-codegen-btreemap-test",
     "test-crates/protobuf-codegen-identical-test",
     "test-crates/protobuf-codegen-protoc-test",
     "test-crates/protobuf-codegen-pure-test",

diff --git a/protobuf-codegen/src/customize/mod.rs b/protobuf-codegen/src/customize/mod.rs
@@ -109,6 +109,8 @@ pub struct Customize {
     /// Used internally to generate protos bundled in protobuf crate
     /// like `descriptor.proto`
     pub(crate) inside_protobuf: Option<bool>,
+    /// When true, protobuf maps are represented with `std::collections::BTreeMap`
+    pub(crate) btreemaps: Option<bool>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -171,6 +173,14 @@ impl Customize {
         self
     }
 
+    /// Use btreemaps for maps representation
+    pub fn btreemaps(self, use_btreemaps: bool) -> Self {
+        Self {
+            btreemaps: Some(use_btreemaps),
+            ..self
+        }
+    }
+
     /// Update fields of self with fields defined in other customize
     pub fn update_with(&mut self, that: &Customize) {
         if let Some(v) = &that.before {
@@ -197,6 +207,9 @@ impl Customize {
         if let Some(v) = that.inside_protobuf {
             self.inside_protobuf = Some(v);
         }
+        if let Some(v) = that.btreemaps {
+            self.btreemaps = Some(v);
+        }
     }
 
     /// Update unset fields of self with fields from other customize

diff --git a/protobuf-codegen/src/customize/rustproto_proto.rs b/protobuf-codegen/src/customize/rustproto_proto.rs
@@ -15,6 +15,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
     let lite_runtime = None;
     let gen_mod_rs = None;
     let inside_protobuf = None;
+    let btreemaps = None;
     Customize {
         before,
         generate_accessors,
@@ -24,6 +25,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
         lite_runtime,
         gen_mod_rs,
         inside_protobuf,
+        btreemaps,
     }
 }
 
@@ -40,6 +42,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
     let lite_runtime = None;
     let gen_mod_rs = None;
     let inside_protobuf = None;
+    let btreemaps = None;
     Customize {
         before,
         generate_accessors,
@@ -49,6 +52,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
         lite_runtime,
         gen_mod_rs,
         inside_protobuf,
+        btreemaps,
     }
 }
 
@@ -61,6 +65,7 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
     let lite_runtime = rustproto::exts::lite_runtime_all.get(source);
     let gen_mod_rs = None;
     let inside_protobuf = None;
+    let btreemaps = None;
     Customize {
         before,
         generate_accessors,
@@ -70,5 +75,6 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
         lite_runtime,
         inside_protobuf,
         gen_mod_rs,
+        btreemaps,
     }
 }
diff --git a/protobuf-codegen/src/gen/field/accessor.rs b/protobuf-codegen/src/gen/field/accessor.rs
@@ -66,7 +66,7 @@ impl FieldGen<'_> {
         let MapField { .. } = map_field;
         AccessorFn {
             name: "make_map_simpler_accessor".to_owned(),
-            type_params: vec![format!("_"), format!("_")],
+            type_params: vec![format!("_")],
             callback_params: self.make_accessor_fns_lambda(),
         }
     }

diff --git a/protobuf-codegen/src/gen/field/mod.rs b/protobuf-codegen/src/gen/field/mod.rs
@@ -276,7 +276,7 @@ impl<'a> FieldGen<'a> {
             FieldKind::Repeated(ref repeated) => repeated.rust_type(reference),
             FieldKind::Map(MapField {
                 ref key, ref value, ..
-            }) => RustType::HashMap(
+            }) => RustType::Map(
                 Box::new(key.rust_storage_elem_type(reference)),
                 Box::new(value.rust_storage_elem_type(reference)),
             ),