Initial layer caching implementation

Makefile

@@ -53,4 +53,7 @@ release:
 
 .PHONY: clean
 clean:
-	rm -rf $(CURDIR)/bin
+	rm -rvf $(CURDIR)/bin
+	rm -rvf $(CURDIR)/*.eopkg
+	rm -rvf $(CURDIR)/pspec_x86_64.xml
+	rm -rvf $(CURDIR)/abi*

builder/build.go

@@ -229,6 +229,24 @@ func (p *Package) CopyAssets(h *PackageHistory, o *Overlay) error {
 	return h.WriteXML(histPath)
 }
 
+func (p *Package) CalcDeps(resolver *Resolver) ([]Dep, error) {
+	// hash = LayersFakeHash
+	extras := []string{}
+
+	if p.HasGitSource() {
+		extras = append(extras, "git")
+	}
+	if p.Clang {
+		extras = append(extras, "llvm-clang-devel")
+	}
+	if p.CanCCache {
+		extras = append(extras, "ccache", "sccache")
+	}
+
+	slog.Debug("Extra dependencyes from recipe", "extras", extras)
+	return resolver.Query(p.Deps, true, true, p.Emul32, extras)
+}
+
 // PrepYpkg will do the initial leg work of preparing us for a ypkg build.
 func (p *Package) PrepYpkg(notif PidNotifier, usr *UserInfo, pman *EopkgManager, overlay *Overlay, h *PackageHistory) error {
 	slog.Debug("Writing packager file")
@@ -246,32 +264,32 @@ func (p *Package) PrepYpkg(notif PidNotifier, usr *UserInfo, pman *EopkgManager,
 		return fmt.Errorf("Failed to write packager file %s, reason: %w\n", fp, err)
 	}
 
-	wdir := p.GetWorkDirInternal()
-	ymlFile := filepath.Join(wdir, filepath.Base(p.Path))
+	// wdir := p.GetWorkDirInternal()
+	// ymlFile := filepath.Join(wdir, filepath.Base(p.Path))
 
-	cmd := fmt.Sprintf("ypkg-install-deps --eopkg-cmd='%s' -f %s", installCommand, ymlFile)
-	if DisableColors {
-		cmd += " -n"
-	}
+	// cmd := fmt.Sprintf("ypkg-install-deps --eopkg-cmd='%s' -f %s", installCommand, ymlFile)
+	// if DisableColors {
+	// 	cmd += " -n"
+	// }
 
-	// Install build dependencies
-	slog.Debug("Installing build dependencies", "file", ymlFile)
+	// // Install build dependencies
+	// slog.Debug("Installing build dependencies", "file", ymlFile)
 
-	if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
-		return fmt.Errorf("Failed to install build dependencies %s, reason: %w\n", ymlFile, err)
-	}
+	// if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
+	// 	return fmt.Errorf("Failed to install build dependencies %s, reason: %w\n", ymlFile, err)
+	// }
 
-	notif.SetActivePID(0)
+	// notif.SetActivePID(0)
 
-	// Cleanup now
-	slog.Debug("Stopping D-BUS")
+	// // Cleanup now
+	// slog.Debug("Stopping D-BUS")
 
-	if err := pman.StopDBUS(); err != nil {
-		return fmt.Errorf("Failed to stop d-bus, reason: %w\n", err)
-	}
+	// if err := pman.StopDBUS(); err != nil {
+	// 	return fmt.Errorf("Failed to stop d-bus, reason: %w\n", err)
+	// }
 
 	// Chwn the directory before bringing up sources
-	cmd = fmt.Sprintf("chown -R %s:%s %s", BuildUser, BuildUser, BuildUserHome)
+	cmd := fmt.Sprintf("chown -R %s:%s %s", BuildUser, BuildUser, BuildUserHome)
 	if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
 		return fmt.Errorf("Failed to set home directory permissions, reason: %w\n", err)
 	}
@@ -492,7 +510,7 @@ func (p *Package) CollectAssets(overlay *Overlay, usr *UserInfo, manifestTarget
 }
 
 // Build will attempt to build the package in the overlayfs system.
-func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, manifestTarget string) error {
+func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, resolver *Resolver, manifestTarget string) error {
 	slog.Debug("Building package", "name", p.Name, "version", p.Version, "release", p.Release, "type", p.Type,
 		"profile", overlay.Back.Name)
 
@@ -513,10 +531,17 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 	}
 
 	// Bring up the root
-	if err := p.ActivateRoot(overlay); err != nil {
+	if err := overlay.ActivateRoot(); err != nil {
 		return err
 	}
 
+	// Add build user if needed
+	if p.Type == PackageTypeYpkg {
+		if err := AddBuildUser(overlay.MountPoint); err != nil {
+			return err
+		}
+	}
+
 	// Ensure source assets are in place
 	if err := p.CopyAssets(history, overlay); err != nil {
 		return fmt.Errorf("Failed to copy required source assets, reason: %w\n", err)
@@ -528,34 +553,34 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 		return err
 	}
 
-	// Set up package manager
-	if err := pman.Init(); err != nil {
-		return err
-	}
+	// // Set up package manager
+	// if err := pman.Init(); err != nil {
+	// 	return err
+	// }
 
-	// Bring up dbus to do Things
-	slog.Debug("Starting D-BUS")
+	// // Bring up dbus to do Things
+	// slog.Debug("Starting D-BUS")
 
-	if err := pman.StartDBUS(); err != nil {
-		return fmt.Errorf("Failed to start d-bus, reason: %w\n", err)
-	}
+	// if err := pman.StartDBUS(); err != nil {
+	// 	return fmt.Errorf("Failed to start d-bus, reason: %w\n", err)
+	// }
 
-	// Get the repos in place before asserting anything
-	if err := p.ConfigureRepos(notif, overlay, pman, profile); err != nil {
-		return fmt.Errorf("Configuring repositories failed, reason: %w\n", err)
-	}
+	// // Get the repos in place before asserting anything
+	// if err := pman.ConfigureRepos(notif, overlay, profile); err != nil {
+	// 	return fmt.Errorf("Configuring repositories failed, reason: %w\n", err)
+	// }
 
-	slog.Debug("Upgrading system base")
+	// slog.Debug("Upgrading system base")
 
-	if err := pman.Upgrade(); err != nil {
-		return fmt.Errorf("Failed to upgrade rootfs, reason: %w\n", err)
-	}
+	// if err := pman.Upgrade(); err != nil {
+	// 	return fmt.Errorf("Failed to upgrade rootfs, reason: %w\n", err)
+	// }
 
-	slog.Debug("Asserting system.devel component installation")
+	// slog.Debug("Asserting system.devel component installation")
 
-	if err := pman.InstallComponent("system.devel"); err != nil {
-		return fmt.Errorf("Failed to assert system.devel, reason: %w\n", err)
-	}
+	// if err := pman.InstallComponent("system.devel"); err != nil {
+	// 	return fmt.Errorf("Failed to assert system.devel, reason: %w\n", err)
+	// }
 
 	// Ensure all directories are in place
 	if err := p.CreateDirs(overlay); err != nil {

builder/chroot.go

@@ -42,7 +42,7 @@ func (p *Package) Chroot(notif PidNotifier, pman *EopkgManager, overlay *Overlay
 
 	ChrootEnvironment = env
 
-	if err := p.ActivateRoot(overlay); err != nil {
+	if err := overlay.ActivateRoot(); err != nil {
 		return err
 	}
 

builder/index.go

@@ -53,7 +53,7 @@ func (p *Package) Index(notif PidNotifier, dir string, overlay *Overlay) error {
 		return err
 	}
 
-	if err := p.ActivateRoot(overlay); err != nil {
+	if err := overlay.ActivateRoot(); err != nil {
 		return err
 	}
 

builder/layer.go

@@ -0,0 +1,178 @@
+package builder
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/btcsuite/btcutil/base58"
+	"github.com/zeebo/blake3"
+)
+
+type Layer struct {
+	deps    []Dep
+	profile *Profile
+	back    *BackingImage
+	// Note: created is only set in `RequestOverlay`
+	created bool
+	hash    string
+}
+
+func (l Layer) MarshalJSON() ([]byte, error) {
+	var imageHash string
+	var err error
+	if PathExists(l.back.ImagePath) {
+		if imageHash, err = xxh3128HashFile(l.back.ImagePath); err != nil {
+			return nil, err
+		}
+		// } else if PathExists(l.back.ImagePath) {
+		// 	if imageHash, err = hashFile(l.back.ImagePath); err != nil {
+		// 		return
+		// 	}
+	} else {
+		return nil, fmt.Errorf("Backing image doens't exist at %s", l.back.ImagePath)
+	}
+
+	return json.Marshal(struct {
+		Deps      []Dep `json:"deps"`
+		ImageHash string
+	}{Deps: l.deps, ImageHash: imageHash})
+}
+
+func (l *Layer) Hash() string {
+	if l.hash == "" {
+		jsonBytes, err := json.Marshal(l)
+		if err != nil {
+			l.hash = LayersFakeHash
+		} else {
+			hashBytes := blake3.Sum256(jsonBytes)
+			l.hash = base58.Encode(hashBytes[:])
+		}
+	}
+	return l.hash
+}
+
+func (l *Layer) BasePath() string {
+	return filepath.Join(LayersDir, l.Hash())
+}
+
+func (l *Layer) RequestOverlay(notif PidNotifier) (contentPath string, err error) {
+	contentPath = filepath.Join(l.BasePath(), "content")
+	if !PathExists(contentPath) || l.Hash() == LayersFakeHash {
+		slog.Info("Creating layer", "hash", l.Hash())
+		return l.Create(notif)
+	} else {
+		l.created = true
+		slog.Info("Reusing layer", "hash", l.Hash())
+		return
+	}
+}
+
+func (l *Layer) RemoveIfNotCreated() error {
+	if !l.created {
+		slog.Info("Removing incomplete layer", "path", l.BasePath())
+		return os.RemoveAll(l.BasePath())
+	} else {
+		return nil
+	}
+}
+
+func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) {
+	basePath := l.BasePath()
+	contentPath = filepath.Join(basePath, "content")
+
+	depsOverlay := Overlay{
+		Back:           l.back,
+		Package:        nil,
+		BaseDir:        basePath,
+		WorkDir:        filepath.Join(basePath, "work"),
+		UpperDir:       contentPath,
+		ImgDir:         filepath.Join(basePath, "img"),
+		MountPoint:     filepath.Join(basePath, "union"),
+		LockPath:       filepath.Join(basePath, "lock"),
+		EnableTmpfs:    false,
+		mountedImg:     false,
+		mountedOverlay: false,
+		mountedVFS:     false,
+		mountedTmpfs:   false,
+	}
+
+	if err = depsOverlay.CleanExisting(); err != nil {
+		err = fmt.Errorf("Failed to cleanup existing overlay (if exists), reason: %w", err)
+		return
+	}
+
+	if err = depsOverlay.EnsureDirs(); err != nil {
+		err = fmt.Errorf("Failed to ensure dirs for overlay, reason: %w", err)
+		return
+	}
+
+	// Mount overlay
+	if err = depsOverlay.ActivateRoot(); err != nil {
+		err = fmt.Errorf("Failed to activate overlay, reason: %w", err)
+		return
+	}
+	defer depsOverlay.DeactivateRoot()
+
+	pman := NewEopkgManager(notif, depsOverlay.MountPoint)
+
+	// Init pman
+	if err = pman.Init(); err != nil {
+		err = fmt.Errorf("Failed to init pman, reason: %w", err)
+		return
+	}
+	defer pman.Cleanup()
+
+	// Bring up dbus to do Things
+	slog.Debug("Starting D-BUS")
+
+	if err = pman.StartDBUS(); err != nil {
+		err = fmt.Errorf("Failed to start d-bus, reason: %w", err)
+		return
+	}
+
+	// Get the repos in place before asserting anything
+	if err = pman.ConfigureRepos(notif, &depsOverlay, l.profile); err != nil {
+		err = fmt.Errorf("Configuring repositories failed, reason: %w", err)
+		return
+	}
+
+	// Now, install/upgrade everything!
+	slog.Debug("Upgrading system base and other core packages")
+
+	if err = pman.Upgrade(); err != nil {
+		err = fmt.Errorf("Failed to upgrade layer rootfs, reason: %w", err)
+		return
+	}
+
+	slog.Debug("Asserting system.devel component installation")
+	if err = pman.InstallComponent("system.devel"); err != nil {
+		err = fmt.Errorf("Failed to assert system.devel in layer, reason: %w", err)
+		return
+	}
+
+	// Install our dependencies
+	pkgs := make([]string, len(l.deps))
+	for idx, dep := range l.deps {
+		pkgs[idx] = dep.Name
+	}
+	slog.Debug("Installing dependencies", "size", len(pkgs), "pkgs", pkgs)
+	cmd := fmt.Sprintf("eopkg it -y %s", strings.Join(pkgs, " "))
+	if DisableColors {
+		cmd += " -n"
+	}
+
+	slog.Debug("Installing dependencies")
+
+	if err = ChrootExec(notif, depsOverlay.MountPoint, cmd); err != nil {
+		err = fmt.Errorf("Failed to install dependencies, reason: %w", err)
+		return
+	}
+	notif.SetActivePID(0)
+
+	l.created = true
+	return
+}

builder/main.go

@@ -47,6 +47,14 @@ const (
 	ImageRootsDir = "/var/lib/solbuild/roots"
 )
 
+const (
+	// LayersDir is where container layers are cached, identified by their
+	// sha256 hashes, e.g. `/var/cache/solbuild/layers/3c0de53d6017469...`
+	LayersDir      = "/var/cache/solbuild/layers"
+	// sha256sum <file> | awk '{ print $1 }' | xxd -r -p | base58
+	LayersFakeHash = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+)
+
 const (
 	// PackageCacheDirectory is where we share packages between all builders.
 	PackageCacheDirectory = "/var/lib/solbuild/packages"