Merge pull request #121 from thaJeztah/integrate_moby_sequential

Integrate github.com/docker/docker/pkg/system Sequential utilities

Makefile

@@ -1,4 +1,4 @@
-PACKAGES ?= mountinfo mount signal symlink
+PACKAGES ?= mountinfo mount sequential signal symlink
 BINDIR ?= _build/bin
 CROSS ?= linux/arm linux/arm64 linux/ppc64le linux/s390x \
 	freebsd/amd64 openbsd/amd64 darwin/amd64 darwin/arm64 windows/amd64

sequential/doc.go

@@ -0,0 +1,15 @@
+// Package sequential provides a set of functions for managing sequential
+// files on Windows.
+//
+// The origin of these functions are the golang OS and windows packages,
+// slightly modified to only cope with files, not directories due to the
+// specific use case.
+//
+// The alteration is to allow a file on Windows to be opened with
+// FILE_FLAG_SEQUENTIAL_SCAN (particular for docker load), to avoid eating
+// the standby list, particularly when accessing large files such as layer.tar.
+//
+// For non-Windows platforms, the package provides wrappers for the equivalents
+// in the os packages. They are passthrough on Unix platforms, and only relevant
+// on Windows.
+package sequential

sequential/go.mod

@@ -0,0 +1,5 @@
+module github.com/moby/sys/sequential
+
+go 1.17
+
+require golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a

sequential/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

sequential/sequential_unix.go

@@ -0,0 +1,45 @@
+//go:build !windows
+// +build !windows
+
+package sequential
+
+import "os"
+
+// Create creates the named file with mode 0666 (before umask), truncating
+// it if it already exists. If successful, methods on the returned
+// File can be used for I/O; the associated file descriptor has mode
+// O_RDWR.
+// If there is an error, it will be of type *PathError.
+func Create(name string) (*os.File, error) {
+	return os.Create(name)
+}
+
+// Open opens the named file for reading. If successful, methods on
+// the returned file can be used for reading; the associated file
+// descriptor has mode O_RDONLY.
+// If there is an error, it will be of type *PathError.
+func Open(name string) (*os.File, error) {
+	return os.Open(name)
+}
+
+// OpenFile is the generalized open call; most users will use Open
+// or Create instead. It opens the named file with specified flag
+// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
+// methods on the returned File can be used for I/O.
+// If there is an error, it will be of type *PathError.
+func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(name, flag, perm)
+}
+
+// CreateTemp creates a new temporary file in the directory dir
+// with a name beginning with prefix, opens the file for reading
+// and writing, and returns the resulting *os.File.
+// If dir is the empty string, TempFile uses the default directory
+// for temporary files (see os.TempDir).
+// Multiple programs calling TempFile simultaneously
+// will not choose the same file. The caller can use f.Name()
+// to find the pathname of the file. It is the caller's responsibility
+// to remove the file when no longer needed.
+func CreateTemp(dir, prefix string) (f *os.File, err error) {
+	return os.CreateTemp(dir, prefix)
+}

sequential/sequential_windows.go

@@ -0,0 +1,161 @@
+package sequential
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+// Create creates the named file with mode 0666 (before umask), truncating
+// it if it already exists. If successful, methods on the returned
+// File can be used for I/O; the associated file descriptor has mode
+// O_RDWR.
+// If there is an error, it will be of type *PathError.
+func Create(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0)
+}
+
+// Open opens the named file for reading. If successful, methods on
+// the returned file can be used for reading; the associated file
+// descriptor has mode O_RDONLY.
+// If there is an error, it will be of type *PathError.
+func Open(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDONLY, 0)
+}
+
+// OpenFile is the generalized open call; most users will use Open
+// or Create instead.
+// If there is an error, it will be of type *PathError.
+func OpenFile(name string, flag int, _ os.FileMode) (*os.File, error) {
+	if name == "" {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
+	}
+	r, err := openFileSequential(name, flag, 0)
+	if err == nil {
+		return r, nil
+	}
+	return nil, &os.PathError{Op: "open", Path: name, Err: err}
+}
+
+func openFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) {
+	r, e := openSequential(name, flag|windows.O_CLOEXEC, 0)
+	if e != nil {
+		return nil, e
+	}
+	return os.NewFile(uintptr(r), name), nil
+}
+
+func makeInheritSa() *windows.SecurityAttributes {
+	var sa windows.SecurityAttributes
+	sa.Length = uint32(unsafe.Sizeof(sa))
+	sa.InheritHandle = 1
+	return &sa
+}
+
+func openSequential(path string, mode int, _ uint32) (fd windows.Handle, err error) {
+	if len(path) == 0 {
+		return windows.InvalidHandle, windows.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		return windows.InvalidHandle, err
+	}
+	var access uint32
+	switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) {
+	case windows.O_RDONLY:
+		access = windows.GENERIC_READ
+	case windows.O_WRONLY:
+		access = windows.GENERIC_WRITE
+	case windows.O_RDWR:
+		access = windows.GENERIC_READ | windows.GENERIC_WRITE
+	}
+	if mode&windows.O_CREAT != 0 {
+		access |= windows.GENERIC_WRITE
+	}
+	if mode&windows.O_APPEND != 0 {
+		access &^= windows.GENERIC_WRITE
+		access |= windows.FILE_APPEND_DATA
+	}
+	sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE)
+	var sa *windows.SecurityAttributes
+	if mode&windows.O_CLOEXEC == 0 {
+		sa = makeInheritSa()
+	}
+	var createmode uint32
+	switch {
+	case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL):
+		createmode = windows.CREATE_NEW
+	case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC):
+		createmode = windows.CREATE_ALWAYS
+	case mode&windows.O_CREAT == windows.O_CREAT:
+		createmode = windows.OPEN_ALWAYS
+	case mode&windows.O_TRUNC == windows.O_TRUNC:
+		createmode = windows.TRUNCATE_EXISTING
+	default:
+		createmode = windows.OPEN_EXISTING
+	}
+	// Use FILE_FLAG_SEQUENTIAL_SCAN rather than FILE_ATTRIBUTE_NORMAL as implemented in golang.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
+	const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN
+	h, e := windows.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0)
+	return h, e
+}
+
+// Helpers for CreateTemp
+var rand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextSuffix() string {
+	randmu.Lock()
+	r := rand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	rand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}
+
+// CreateTemp is a copy of os.CreateTemp, modified to use sequential
+// file access. Below is the original comment from golang:
+// TempFile creates a new temporary file in the directory dir
+// with a name beginning with prefix, opens the file for reading
+// and writing, and returns the resulting *os.File.
+// If dir is the empty string, TempFile uses the default directory
+// for temporary files (see os.TempDir).
+// Multiple programs calling TempFile simultaneously
+// will not choose the same file. The caller can use f.Name()
+// to find the pathname of the file. It is the caller's responsibility
+// to remove the file when no longer needed.
+func CreateTemp(dir, prefix string) (f *os.File, err error) {
+	if dir == "" {
+		dir = os.TempDir()
+	}
+
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+nextSuffix())
+		f, err = OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
+		if os.IsExist(err) {
+			if nconflict++; nconflict > 10 {
+				randmu.Lock()
+				rand = reseed()
+				randmu.Unlock()
+			}
+			continue
+		}
+		break
+	}
+	return
+}