diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go
index 912a878bf6d..78538c8594f 100644
--- a/driver/docker-container/driver.go
+++ b/driver/docker-container/driver.go
@@ -37,6 +37,7 @@ const (
 type Driver struct {
 	driver.InitConfig
 	factory      driver.Factory
+	userNSRemap  bool // true if dockerd is running with userns-remap mode
 	netMode      string
 	image        string
 	cgroupParent string
@@ -112,7 +113,6 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 	if err := l.Wrap("creating container "+d.Name, func() error {
 		hc := &container.HostConfig{
 			Privileged: true,
-			UsernsMode: "host",
 			Mounts: []mount.Mount{
 				{
 					Type:   mount.TypeVolume,
@@ -121,6 +121,9 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 				},
 			},
 		}
+		if d.userNSRemap {
+			hc.UsernsMode = "host"
+		}
 		if d.netMode != "" {
 			hc.NetworkMode = container.NetworkMode(d.netMode)
 		}
diff --git a/driver/docker-container/factory.go b/driver/docker-container/factory.go
index ae4d39b9a47..9b8212fd918 100644
--- a/driver/docker-container/factory.go
+++ b/driver/docker-container/factory.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/docker/buildx/driver"
+	dockertypes "github.com/docker/docker/api/types"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 )
@@ -40,6 +41,20 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		return nil, errors.Errorf("%s driver requires docker API access", f.Name())
 	}
 	d := &Driver{factory: f, InitConfig: cfg}
+	dockerInfo, err := cfg.DockerAPI.Info(ctx)
+	if err != nil {
+		return nil, err
+	}
+	secOpts, err := dockertypes.DecodeSecurityOptions(dockerInfo.SecurityOptions)
+	if err != nil {
+		return nil, err
+	}
+	for _, f := range secOpts {
+		if f.Name == "userns" {
+			d.userNSRemap = true
+			break
+		}
+	}
 	for k, v := range cfg.DriverOpts {
 		switch {
 		case k == "network":