diff --git a/script/travisbuild b/script/travisbuild
index 152606e25..5c83d1116 100755
--- a/script/travisbuild
+++ b/script/travisbuild
@@ -26,6 +26,4 @@ echo 'PasswordAuthentication yes' | sudo tee -a /etc/sshd_config
 eval $(ssh-agent)
 ssh-add $GITTEST_REMOTE_SSH_KEY
 
-# We need the config so the tests don't fail
-git config --global user.name 'The rugged tests are fragile'
 bundle exec rake || exit $?
diff --git a/test/config_test.rb b/test/config_test.rb
index 99ea9b032..2e4362a83 100644
--- a/test/config_test.rb
+++ b/test/config_test.rb
@@ -3,6 +3,16 @@
 class ConfigTest < Rugged::TestCase
   def setup
     @repo = FixtureRepo.from_rugged("testrepo.git")
+
+    path = Dir.mktmpdir("rugged-global-config")
+    cfg = Rugged::Config.new(File.join(path, ".gitconfig"))
+    cfg['user.name'] = "The test suite"
+    Rugged::Settings['search_path_global'] = path
+    @glocalconfigdir = path
+  end
+
+  def cleanup
+    FileUtils.remove_entry_secure(@globalconfigdir)
   end
 
   def test_multi_fetch
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 196c145e6..e66a9cad5 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -6,9 +6,22 @@
 
 module Rugged
   class TestCase < Minitest::Test
+    # Set up some isolation for our tests so we don't try to touch any
+    # configuration from the user running the tests
+    def before_setup
+      @configdir ||= begin
+        path = Dir.mktmpdir("rugged-config")
+        Rugged::Settings['search_path_global'] = path
+        Rugged::Settings['search_path_xdg'] = path
+        Rugged::Settings['search_path_system'] = path
+      end
+      super
+    end
+
     # Automatically clean up created fixture repos after each test run
     def after_teardown
       Rugged::TestCase::FixtureRepo.teardown
+      FileUtils.remove_entry_secure(@configdir)
       super
     end