chunked ApplyDiff(): track implicitly-created directories in layers

When applying a chunked diff, ApplyDiff() iterates through the entries
in the diff's table of contents, creating directories and other
zero-length items like empty files, symbolic links, and directories,
building a queue of hard links, and asking goroutines to locate files in
other layers and configured OSTree repositories which contain content
that matches what should eventually end up in non-zero-length files in
the layer.

If a file's entire contents couldn't be found locally, but it was split
into chunks in the chunked diff, ApplyDiff() then searches for the
chunks in local layers.

Lastly, the contents of files with non-zero lengths, which ApplyDiff()
had attempted to find locally, but which weren't found locally, are then
requested from the registry.

Track which directories in the layer are created implicitly by
ApplyDiff(), so that the overlay driver can reset their ownership,
permissions, extended attributes, and datestamps to match the
immediately lower layer, if there is one.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

drivers/driver.go

@@ -188,6 +188,7 @@ type DriverWithDifferOutput struct {
 	Metadata           string
 	BigData            map[string][]byte
 	TarSplit           []byte
+	ImplicitDirs       []string
 	TOCDigest          digest.Digest
 }
 

drivers/overlay/overlay.go

@@ -1955,6 +1955,49 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri
 		return err
 	}
 
+	lowerDiffDirs, err := d.getLowerDiffPaths(id)
+	if err != nil {
+		return err
+	}
+	if len(lowerDiffDirs) > 0 {
+		backfiller := unionbackfill.NewBackfiller(options.Mappings, lowerDiffDirs)
+		for _, implicitDir := range diffOutput.ImplicitDirs {
+			hdr, err := backfiller.Backfill(implicitDir)
+			if err != nil {
+				return err
+			}
+			if hdr == nil {
+				continue
+			}
+			path := filepath.Join(stagingDirectory, implicitDir)
+			idPair := idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
+			if options.Mappings != nil {
+				if mapped, err := options.Mappings.ToHost(idPair); err == nil {
+					idPair = mapped
+				}
+			}
+			if err := os.Chown(path, idPair.UID, idPair.GID); err != nil {
+				return err
+			}
+			for xattr, xval := range hdr.Xattrs {
+				if err := system.Lsetxattr(path, xattr, []byte(xval), 0); err != nil {
+					return err
+				}
+			}
+			if err := os.Chmod(path, os.FileMode(hdr.Mode)&os.ModePerm); err != nil {
+				return err
+			}
+			atime := hdr.AccessTime
+			mtime := hdr.ModTime
+			if atime.IsZero() {
+				atime = mtime
+			}
+			if err := os.Chtimes(path, atime, mtime); err != nil {
+				return err
+			}
+		}
+	}
+
 	diffOutput.UncompressedDigest = diffOutput.TOCDigest
 
 	return os.Rename(stagingDirectory, diff)

pkg/chunked/cache_linux.go

@@ -109,7 +109,7 @@ func (c *layersCache) load() error {
 		}
 
 		bigData, err := c.store.LayerBigData(r.ID, cacheKey)
-		// if the cache areadly exists, read and use it
+		// if the cache already exists, read and use it
 		if err == nil {
 			defer bigData.Close()
 			metadata, err := readMetadataFromCache(bigData)

pkg/chunked/internal/compression.go

@@ -48,7 +48,7 @@ type FileMetadata struct {
 	ChunkDigest string `json:"chunkDigest,omitempty"`
 	ChunkType   string `json:"chunkType,omitempty"`
 
-	// internal: computed by mergeTOCEntries.
+	// internal: computed by mergeTocEntries.
 	Chunks []*FileMetadata `json:"-"`
 }