Unable to proxy or mirror images using Dragonfly2

[PKizzle]

Version of nydus being used (nydusd --version)

Version: v2.3.0-alpha.0
Git Commit: d7b1851
Build Time: 2023-09-27T06:54:04.052591588Z
Profile: release
Rustc: rustc 1.68.2 (9eb3afe9e 2023-03-27)

Version of nydus-snapshotter being used (containerd-nydus-grpc --version)

Version: v0.13.1-23-g5009c52.m
Revision: 5009c522df583cdf76ee37ae2a3c6440d5d79797.m
Go version: go1.21.4
Build time: 2023-11-10T03:56:00

Kernel information (uname -r)

6.1.54-v8+ aarch64 GNU/Linux

GNU/Linux Distribution, if applicable (cat /etc/os-release)

PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
NAME="Debian GNU/Linux"
VERSION_ID="12"
VERSION="12 (bookworm)"
VERSION_CODENAME=bookworm
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"

containerd-nydus-grpc config

version = 1
# Snapshotter's own home directory where it stores and creates necessary resources
root = "/var/lib/rancher/k3s/agent/containerd/nydus/snapshotter"
# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
daemon_mode = "dedicated"
# Whether snapshotter should try to clean up resources when it is closed
cleanup_on_close = true

[system]
# Snapshotter's debug and trace HTTP server interface
enable = false
# Unix domain socket path where system controller is listening on
address = "/var/lib/containerd-nydus/system.sock"

[system.debug]
# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started.
# This option specifies the profile duration when nydusd is downloading and uncomproessing data.
daemon_cpu_profile_duration_secs = 0
# Enable by assigning an address, empty indicates pprof server is disabled
pprof_address = ""

[daemon]
nydusd_path = "/usr/local/bin/nydusd"
nydusimage_path = "/usr/local/bin/nydus-image"
# fusedev or fscache
fs_driver = "fusedev"
# Specify nydusd log level
log_level = "warn"
# How to process when daemon dies: "none", "restart" or "failover"
recover_policy = "restart"
# Specify a configuration file for nydusd
nydusd_config = "/etc/nydus/nydusd-config.fusedev.json"
# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024].
# Setting to 0 will use the default configuration of nydusd.
threads_number = 0
# Log rotation size for nydusd, in unit MB(megabytes)
log_rotation_size = 2

[cgroup]
# Whether to use separate cgroup for nydusd.
enable = true
# The memory limit for nydusd cgroup, which contains all nydusd processes.
# Percentage is supported as well, please ensure it is end with "%".
# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".
memory_limit = ""

[log]
# Print logs to stdout rather than logging files
log_to_stdout = true
# Snapshotter's log level
level = "warn"
log_rotation_compress = true
log_rotation_local_time = true
# Max number of days to retain logs
log_rotation_max_age = 7
log_rotation_max_backups = 2
# In unit MB(megabytes)
log_rotation_max_size = 2

[metrics]
# Enable by assigning an address, empty indicates metrics server is disabled
address = ""

[remote]
convert_vpc_registry = false

[remote.mirrors_config]
# Snapshotter will overwrite daemon's mirrors configuration
# if the values loaded from this directory are not null before starting a daemon.
# Set to "" or an empty directory to disable it.
# "/etc/nydus/certs.d"
# "/var/lib/rancher/k3s/agent/etc/containerd/certs.d"
dir = ""

[remote.auth]
# Fetch the private registry auth by listening to K8s API server
enable_kubeconfig_keychain = false
# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`)
kubeconfig_path = ""
# Fetch the private registry auth as CRI image service proxy
enable_cri_keychain = true
# the target image service when using image proxy
image_service_address = "/run/k3s/containerd/containerd.sock"

[snapshot]
# Let containerd use nydus-overlayfs mount helper
enable_nydus_overlayfs = true
# Insert Kata Virtual Volume option to `Mount.Options`
enable_kata_volume = false
# Whether to remove resources when a snapshot is removed
sync_remove = true

[cache_manager]
disable = false
gc_period = "6h"
cache_dir = "/var/lib/rancher/k3s/agent/containerd/nydus/blobcache/"

[image]
public_key_file = ""
validate_signature = false

# The configuraions for features that are not production ready
[experimental]
# Whether to enable stargz support
enable_stargz = false
# Whether to enable referrers support
# The option enables trying to fetch the Nydus image associated with the OCI image and run it.
# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
enable_referrer_detect = false
[experimental.tarfs]
# Whether to enable nydus tarfs mode. Tarfs is supported by:
# - The EROFS filesystem driver since Linux 6.4
# - Nydus Image Service release v2.3
enable_tarfs = false
# Mount rafs on host by loopdev and EROFS
mount_tarfs_on_host = false
# Only enable nydus tarfs mode for images with `tarfs hint` label when true
tarfs_hint = false
# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
max_concurrent_proc = 0
# Mode to export tarfs images:
# - "none" or "": do not export tarfs
# - "layer_verity_only": only generate disk verity information for a layer blob
# - "image_verity_only": only generate disk verity information for all blobs of an image
# - "layer_block": generate a raw block disk image with tarfs for a layer
# - "image_block": generate a raw block disk image with tarfs for an image
# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
export_mode = "layer_block_with_verity"

nydusd config

{
  "device": {
    "backend": {
      "type": "registry",
      "config": {
        "mirrors": [
          {
            "host": "http://127.0.0.1:65001",
            "headers": {
              "X-Dragonfly-Registry": "https://index.docker.io"
            },
            "ping_url": "http://127.0.0.1:40901/server/ping",
            "health_check_interval": 5,
            "failure_limit": 3
          }
        ],
        "skip_verify": false,
        "timeout": 10,
        "connect_timeout": 10,
        "retry_limit": 2
      }
    },
    "cache": {
      "type": "filecache",
      "validate": true,
      "config": {
        "work_dir": "/var/lib/rancher/k3s/agent/containerd/nydus/blobcache/",
        "enable_encryption": false
      }
    }
  },
  "mode": "direct",
  "digest_validate": false,
  "iostats_files": false,
  "enable_xattr": true,
  "access_pattern": false,
  "latest_read_files": false,
  "fs_prefetch": {
    "enable": true,
    "prefetch_all": false,
    "threads_count": 2,
    "merging_size": 100000,
    "bandwidth_rate": 0
  }
}

client command line used, if applicable (such as: nerdctl, docker, kubectl, ctr)

I am starting a container in a Kubernetes cluster. Maybe similar behavior can be achieved by running:

nerdctl run -ti --rm --entrypoint=/bin/sh docker.io/thegrandpkizzle/authentik:2023.10.2-1c68198-nydus

The image has been converted using:

nydusify convert --source thegrandpkizzle/authentik:2023.10.2-1c68198 --target docker.io/thegrandpkizzle/authentik:2023.10.2-1c68198-nydus --all-platforms --fs-version 6 --prefetch-patterns < nydus-prefetch.txt

Details about issue

Mirroring or proxying OCIv1 images to Dragonfly2 works as expected. However, when using nydus in either mirror or proxy mode I receive a "os error 22" linked to invalid mount options.

DEBUG [/src/backend/connection.rs:673] cache-flusher Request: GET http://127.0.0.1:65001/v2/thegrandpkizzle/authentik/blobs/sha256:8bd18e4229771b0294a8a0214d6b7a76fa25176f0f497fcc1b3a4cc442e778c7 headers: Some({"range": "bytes=3343185-3349338", "x-dragonfly-registry": "https://index.docker.io"}), proxy: false, data: false, duration: 498ms
DEBUG [/src/backend/connection.rs:539] [mirror] replace to: http://127.0.0.1:65001/token
DEBUG [/src/backend/connection.rs:673] cache-flusher Request: POST http://127.0.0.1:65001/token headers: Some({"x-dragonfly-registry": "https://index.docker.io"}), proxy: false, data: true, duration: 377ms
WARN [/src/backend/mod.rs:132] Read from backend failed: Registry(Common("Invalid argument (os error 22)")), retry count 2
DEBUG [/src/backend/connection.rs:539] [mirror] replace to: http://127.0.0.1:65001/v2/thegrandpkizzle/authentik/blobs/sha256:78b7f9b47ded9a51cf9d71b61da95643fe99473cef0babac3a19d5b3443b8f24
level=info msg="[Prepare] snapshot with key k8s.io/7509/d03c06755cd386e28464b494fc581ed0e81bb27535d108e8037cc545d2999c9c parent k8s.io/6103/sha256:fdd1806c0c904b645baa46ed5dd5d7ba6d7fb773779d2c8263a32716b5ef6700"
level=debug msg="[Prepare] snapshot with labels map[]" key=k8s.io/7509/d03c06755cd386e28464b494fc581ed0e81bb27535d108e8037cc545d2999c9c parent="k8s.io/6103/sha256:fdd1806c0c904b645baa46ed5dd5d7ba6d7fb773779d2c8263a32716b5ef6700"
level=info msg="Prepare active Nydus snapshot k8s.io/7509/d03c06755cd386e28464b494fc581ed0e81bb27535d108e8037cc545d2999c9c" key=k8s.io/7509/d03c06755cd386e28464b494fc581ed0e81bb27535d108e8037cc545d2999c9c parent="k8s.io/6103/sha256:fdd1806c0c904b645baa46ed5dd5d7ba6d7fb773779d2c8263a32716b5ef6700"
level=debug msg="Prepare remote snapshot 5745" key=k8s.io/7509/d03c06755cd386e28464b494fc581ed0e81bb27535d108e8037cc545d2999c9c parent="k8s.io/6103/sha256:fdd1806c0c904b645baa46ed5dd5d7ba6d7fb773779d2c8263a32716b5ef6700"
level=info msg="nydusd command: /usr/local/bin/nydusd fuse --thread-num 4 --config /var/lib/rancher/k3s/agent/containerd/nydus/snapshotter/config/cl9d8r9m8rbd0e3g3m40/config.json --bootstrap /var/lib/rancher/k3s/agent/containerd/nydus/snapshotter/snapshots/5745/fs/image/image.boot --mountpoint /var/lib/rancher/k3s/agent/containerd/nydus/snapshotter/snapshots/5745/mnt --apisock /var/lib/rancher/k3s/agent/containerd/nydus/snapshotter/socket/cl9d8r9m8rbd0e3g3m40/api.sock --log-level debug --log-rotation-size 2"
INFO Program Version: v2.3.0-alpha.0, Git Commit: "d7b1851f425ccbdd57a3ac9574992545b335b9c0", Build Time: "2023-09-27T06:54:04.052591588Z", Profile: "release", Rustc Version: "rustc 1.68.2 (9eb3afe9e 2023-03-27)"
INFO Set rlimit-nofile to 1000000, maximum 1000000
DEBUG [/fuse-backend-rs-0.10.5/src/api/pseudo_fs.rs:161] pseudo fs iterate "/"
INFO RAFS features: HASH_BLAKE3 | EXPLICIT_UID_GID | HAS_XATTR | COMPRESSION_ZSTD | INLINED_CHUNK_DIGEST | ENCRYPTION_NONE
INFO backend config: ConnectionConfig { proxy: ProxyConfig { url: "", ping_url: "", fallback: false, check_interval: 5, use_http: false }, mirrors: [MirrorConfig { host: "http://127.0.0.1:65001", ping_url: "", headers: {"X-Dragonfly-Registry": "https://index.docker.io"}, health_check_interval: 5, failure_limit: 5 }, MirrorConfig { host: "https://index.docker.io", ping_url: "", headers: {}, health_check_interval: 5, failure_limit: 5 }], skip_verify: false, timeout: 10, connect_timeout: 10, retry_limit: 2 }
INFO [mirror] start health check, ping url: http://127.0.0.1:65001/v2
INFO [mirror] start health check, ping url: https://index.docker.io/v2
INFO Refresh token thread started.
DEBUG [/src/backend/connection.rs:539] [mirror] replace to: http://127.0.0.1:65001/v2/thegrandpkizzle/pgbouncer/blobs/sha256:1514af95786fbd9a1bb1eff4a75f9ec1d79c0ef2924ada18695ac8499e5bccc3
INFO RAFS filesystem imported
INFO Rafs filesystem mounted at /
INFO mount source rafs dest /var/lib/rancher/k3s/agent/containerd/nydus/snapshotter/snapshots/5745/mnt with fstype fuse opts default_permissions,allow_other,fd=44,rootmode=40000,user_id=0,group_id=0 fd 44
DEBUG [/reqwest-0.11.16/src/connect.rs:429] starting new connection: http://127.0.0.1:65001/
INFO State machine(pid=1836675): from Init to Ready, input [Mount], output [None]
INFO State machine(pid=1836675): from Ready to Running, input [Start], output [Some(StartService)]
INFO start fuse servers with 4 worker threads
INFO FUSE INIT major 7 minor 37
Nov 14 02:59:09 raspberrypi-4-2 containerd-nydus-grpc[1836675]:  in_opts: ASYNC_READ | POSIX_LOCKS | ATOMIC_O_TRUNC | EXPORT_SUPPORT | BIG_WRITES | DONT_MASK | SPLICE_WRITE | SPLICE_MOVE | SPLICE_READ | FLOCK_LOCKS | HAS_IOCTL_DIR | AUTO_INVAL_DATA | DO_READDIRPLUS | READDIRPLUS_AUTO | ASYNC_DIO | WRITEBACK_CACHE | ZERO_MESSAGE_OPEN | PARALLEL_DIROPS | HANDLE_KILLPRIV | POSIX_ACL | ABORT_ERROR | MAX_PAGES | CACHE_SYMLINKS | ZERO_MESSAGE_OPENDIR | EXPLICIT_INVAL_DATA | HANDLE_KILLPRIV_V2 | INIT_EXT
Nov 14 02:59:09 raspberrypi-4-2 containerd-nydus-grpc[1836675]: out_opts: ASYNC_READ | BIG_WRITES | HAS_IOCTL_DIR | AUTO_INVAL_DATA | DO_READDIRPLUS | READDIRPLUS_AUTO | ASYNC_DIO | WRITEBACK_CACHE | ZERO_MESSAGE_OPEN | PARALLEL_DIROPS | MAX_PAGES | CACHE_SYMLINKS | ZERO_MESSAGE_OPENDIR | EXPLICIT_INVAL_DATA
INFO Fuse daemon started!

[imeoer]

cc @lihuahua123

[lyj199907]

I will try it.

[PKizzle]

@lyj199907 Do you require any further information in order to reproduce the issue?

[imeoer]

@PKizzle It seems @lyj199907 hasn't had time to look at it yet, I'll take it later.