diff --git a/bundler/lib/bundler/source/git/git_proxy.rb b/bundler/lib/bundler/source/git/git_proxy.rb
index 180fb210ffa1..5dc1c8de32d7 100644
--- a/bundler/lib/bundler/source/git/git_proxy.rb
+++ b/bundler/lib/bundler/source/git/git_proxy.rb
@@ -136,7 +136,9 @@ def copy_to(destination, submodules = false)
           if submodules
             git_retry "submodule update --init --recursive", :dir => destination
           elsif Gem::Version.create(version) >= Gem::Version.create("2.9.0")
-            git_retry 'submodule foreach --quiet "git -C \$toplevel submodule deinit --force \$sm_path"', :dir => destination
+            inner_command = "git -C $toplevel submodule deinit --force $sm_path"
+            inner_command = inner_command.gsub("$") { '\$' } unless Bundler::WINDOWS
+            git_retry "submodule foreach --quiet \"#{inner_command}\"", :dir => destination
           end
         end