feat(add): add --dereference-symlinks, --empty-dirs, --hidden CLI flags

add CLI flags for controlling file collection behavior during ipfs add:

- `--dereference-symlinks`: recursively resolve symlinks to their target
  content (replaces deprecated --dereference-args which only worked on
  CLI arguments). wired through go-ipfs-cmds to boxo's SerialFileOptions.
- `--empty-dirs` / `-E`: include empty directories (default: true)
- `--hidden` / `-H`: include hidden files (default: false)

these flags are CLI-only and not wired to Import.* config options because
go-ipfs-cmds library handles input file filtering before the directory
tree is passed to kubo. removed unused Import.UnixFSSymlinkMode config
option that was defined but never actually read by the CLI.

also:
- wire --trickle to Import.UnixFSDAGLayout config default
- update go-ipfs-cmds to v0.15.1-0.20260117043932-17687e216294
- add SYMLINK HANDLING section to ipfs add help text
- add CLI tests for all three flags

ref: ipfs/specs#499

Author: lidel

config/import.go

@@ -35,17 +35,13 @@ const (
 	HAMTSizeEstimationBlock    = "block"    // full serialized dag-pb block size
 	HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely
 
-	// SymlinkMode values for Import.UnixFSSymlinkMode
-	SymlinkModePreserve    = "preserve"    // preserve symlinks as UnixFS symlink nodes
-	SymlinkModeDereference = "dereference" // dereference symlinks, import target content
-
 	// DAGLayout values for Import.UnixFSDAGLayout
 	DAGLayoutBalanced = "balanced" // balanced DAG layout (default)
 	DAGLayoutTrickle  = "trickle"  // trickle DAG layout
 
 	DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior
-	DefaultUnixFSSymlinkMode                 = SymlinkModePreserve     // preserve symlinks as UnixFS symlink nodes
 	DefaultUnixFSDAGLayout                   = DAGLayoutBalanced       // balanced DAG layout
+	DefaultUnixFSIncludeEmptyDirs            = true                    // include empty directories
 )
 
 var (
@@ -66,7 +62,6 @@ type Import struct {
 	UnixFSHAMTDirectoryMaxFanout      OptionalInteger
 	UnixFSHAMTDirectorySizeThreshold  OptionalBytes
 	UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled"
-	UnixFSSymlinkMode                 OptionalString // "preserve" or "dereference"
 	UnixFSDAGLayout                   OptionalString // "balanced" or "trickle"
 	BatchMaxNodes                     OptionalInteger
 	BatchMaxSize                      OptionalInteger
@@ -161,18 +156,6 @@ func ValidateImportConfig(cfg *Import) error {
 		}
 	}
 
-	// Validate UnixFSSymlinkMode
-	if !cfg.UnixFSSymlinkMode.IsDefault() {
-		mode := cfg.UnixFSSymlinkMode.WithDefault(DefaultUnixFSSymlinkMode)
-		switch mode {
-		case SymlinkModePreserve, SymlinkModeDereference:
-			// valid
-		default:
-			return fmt.Errorf("Import.UnixFSSymlinkMode must be %q or %q, got %q",
-				SymlinkModePreserve, SymlinkModeDereference, mode)
-		}
-	}
-
 	// Validate UnixFSDAGLayout
 	if !cfg.UnixFSDAGLayout.IsDefault() {
 		layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout)

config/import_test.go

@@ -446,43 +446,6 @@ func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) {
 	}
 }
 
-func TestValidateImportConfig_SymlinkMode(t *testing.T) {
-	tests := []struct {
-		name    string
-		value   string
-		wantErr bool
-		errMsg  string
-	}{
-		{name: "valid preserve", value: SymlinkModePreserve, wantErr: false},
-		{name: "valid dereference", value: SymlinkModeDereference, wantErr: false},
-		{name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"},
-		{name: "invalid empty", value: "", wantErr: true, errMsg: "must be"},
-		{name: "invalid follow", value: "follow", wantErr: true, errMsg: "must be"},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			cfg := &Import{
-				UnixFSSymlinkMode: *NewOptionalString(tt.value),
-			}
-
-			err := ValidateImportConfig(cfg)
-
-			if tt.wantErr {
-				if err == nil {
-					t.Errorf("expected error for value=%q, got nil", tt.value)
-				} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
-					t.Errorf("error = %v, want error containing %q", err, tt.errMsg)
-				}
-			} else {
-				if err != nil {
-					t.Errorf("unexpected error for value=%q: %v", tt.value, err)
-				}
-			}
-		})
-	}
-}
-
 func TestValidateImportConfig_DAGLayout(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -528,7 +491,7 @@ func TestImport_HAMTSizeEstimationMode(t *testing.T) {
 		{HAMTSizeEstimationLinks, io.SizeEstimationLinks},
 		{HAMTSizeEstimationBlock, io.SizeEstimationBlock},
 		{HAMTSizeEstimationDisabled, io.SizeEstimationDisabled},
-		{"", io.SizeEstimationLinks},       // default (unset returns default)
+		{"", io.SizeEstimationLinks},        // default (unset returns default)
 		{"unknown", io.SizeEstimationLinks}, // fallback to default
 	}
 

config/profile.go

@@ -338,7 +338,6 @@ See https://github.com/ipfs/specs/pull/499`,
 			c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
 			c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB")
 			c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock)
-			c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve)
 			c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced)
 			return nil
 		},
@@ -436,7 +435,6 @@ func applyUnixFSv02015(c *Config) error {
 	c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
 	c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB")
 	c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks)
-	c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve)
 	c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced)
 	return nil
 }

core/commands/add.go

@@ -68,6 +68,7 @@ const (
 	mtimeNsecsOptionName      = "mtime-nsecs"
 	fastProvideRootOptionName = "fast-provide-root"
 	fastProvideWaitOptionName = "fast-provide-wait"
+	emptyDirsOptionName       = "empty-dirs"
 )
 
 const (
@@ -147,6 +148,18 @@ to find it in the future:
 See 'ipfs files --help' to learn more about using MFS
 for keeping track of added files and directories.
 
+SYMLINK HANDLING:
+
+By default, symbolic links are preserved as UnixFS symlink nodes that store
+the target path. Use --dereference-symlinks to resolve symlinks to their
+target content instead:
+
+  > ipfs add -r --dereference-symlinks ./mydir
+
+This recursively resolves all symlinks encountered during directory traversal.
+Symlinks to files become regular file content, symlinks to directories are
+traversed and their contents are added.
+
 CHUNKING EXAMPLES:
 
 The chunker option, '-s', specifies the chunking strategy that dictates
@@ -200,11 +213,13 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 	Options: []cmds.Option{
 		// Input Processing
 		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
-		cmds.OptionDerefArgs,     // a builtin option that resolves passed in filesystem links (--dereference-args)
+		cmds.OptionDerefArgs,     // DEPRECATED: use --dereference-symlinks instead
 		cmds.OptionStdinName,     // a builtin option that optionally allows wrapping stdin into a named file
 		cmds.OptionHidden,
 		cmds.OptionIgnore,
 		cmds.OptionIgnoreRules,
+		cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs),
+		cmds.OptionDerefSymlinks, // resolve symlinks to their target content
 		// Output Control
 		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
 		cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."),
@@ -274,7 +289,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 		}
 
 		progress, _ := req.Options[progressOptionName].(bool)
-		trickle, _ := req.Options[trickleOptionName].(bool)
+		trickle, trickleSet := req.Options[trickleOptionName].(bool)
 		wrap, _ := req.Options[wrapOptionName].(bool)
 		onlyHash, _ := req.Options[onlyHashOptionName].(bool)
 		silent, _ := req.Options[silentOptionName].(bool)
@@ -312,6 +327,19 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 		mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
 		fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
 		fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
+		emptyDirs, _ := req.Options[emptyDirsOptionName].(bool)
+
+		// Handle --dereference-args deprecation
+		derefArgs, derefArgsSet := req.Options[cmds.DerefLong].(bool)
+		if derefArgsSet && derefArgs {
+			return fmt.Errorf("--dereference-args is deprecated: use --dereference-symlinks instead")
+		}
+
+		// Wire --trickle from config
+		if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() {
+			layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout)
+			trickle = layout == config.DAGLayoutTrickle
+		}
 
 		if chunker == "" {
 			chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
@@ -409,6 +437,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 
 			options.Unixfs.PreserveMode(preserveMode),
 			options.Unixfs.PreserveMtime(preserveMtime),
+
+			options.Unixfs.IncludeEmptyDirs(emptyDirs),
 		}
 
 		if mode != 0 {

core/coreapi/unixfs.go

@@ -183,6 +183,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	fileAdder.PreserveMtime = settings.PreserveMtime
 	fileAdder.FileMode = settings.Mode
 	fileAdder.FileMtime = settings.Mtime
+	if settings.IncludeEmptyDirsSet {
+		fileAdder.IncludeEmptyDirs = settings.IncludeEmptyDirs
+	}
 
 	switch settings.Layout {
 	case options.BalancedLayout:

core/coreiface/options/unixfs.go

@@ -48,10 +48,12 @@ type UnixfsAddSettings struct {
 	Silent   bool
 	Progress bool
 
-	PreserveMode  bool
-	PreserveMtime bool
-	Mode          os.FileMode
-	Mtime         time.Time
+	PreserveMode        bool
+	PreserveMtime       bool
+	Mode                os.FileMode
+	Mtime               time.Time
+	IncludeEmptyDirs    bool
+	IncludeEmptyDirsSet bool
 }
 
 type UnixfsLsSettings struct {
@@ -93,10 +95,12 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix,
 		Silent:   false,
 		Progress: false,
 
-		PreserveMode:  false,
-		PreserveMtime: false,
-		Mode:          0,
-		Mtime:         time.Time{},
+		PreserveMode:        false,
+		PreserveMtime:       false,
+		Mode:                0,
+		Mtime:               time.Time{},
+		IncludeEmptyDirs:    true, // default: include empty directories
+		IncludeEmptyDirsSet: false,
 	}
 
 	for _, opt := range opts {
@@ -396,3 +400,12 @@ func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption {
 		return nil
 	}
 }
+
+// IncludeEmptyDirs tells the adder to include empty directories in the DAG
+func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.IncludeEmptyDirs = include
+		settings.IncludeEmptyDirsSet = true
+		return nil
+	}
+}

core/coreunix/add.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/ipfs/kubo/config"
 	coreiface "github.com/ipfs/kubo/core/coreiface"
 
 	"github.com/ipfs/kubo/tracing"
@@ -52,17 +53,18 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG
 	bufferedDS := ipld.NewBufferedDAG(ctx, ds)
 
 	return &Adder{
-		ctx:           ctx,
-		pinning:       p,
-		gcLocker:      bs,
-		dagService:    ds,
-		bufferedDS:    bufferedDS,
-		Progress:      false,
-		Pin:           true,
-		Trickle:       false,
-		MaxLinks:      ihelper.DefaultLinksPerBlock,
-		MaxHAMTFanout: uio.DefaultShardWidth,
-		Chunker:       "",
+		ctx:              ctx,
+		pinning:          p,
+		gcLocker:         bs,
+		dagService:       ds,
+		bufferedDS:       bufferedDS,
+		Progress:         false,
+		Pin:              true,
+		Trickle:          false,
+		MaxLinks:         ihelper.DefaultLinksPerBlock,
+		MaxHAMTFanout:    uio.DefaultShardWidth,
+		Chunker:          "",
+		IncludeEmptyDirs: config.DefaultUnixFSIncludeEmptyDirs,
 	}, nil
 }
 
@@ -91,10 +93,11 @@ type Adder struct {
 	CidBuilder        cid.Builder
 	liveNodes         uint64
 
-	PreserveMode  bool
-	PreserveMtime bool
-	FileMode      os.FileMode
-	FileMtime     time.Time
+	PreserveMode     bool
+	PreserveMtime    bool
+	FileMode         os.FileMode
+	FileMtime        time.Time
+	IncludeEmptyDirs bool
 }
 
 func (adder *Adder) mfsRoot() (*mfs.Root, error) {
@@ -480,6 +483,24 @@ func (adder *Adder) addFile(path string, file files.File) error {
 func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error {
 	log.Infof("adding directory: %s", path)
 
+	// Peek at first entry to check if directory is empty.
+	// We advance the iterator once here and continue from this position
+	// in the processing loop below. This avoids allocating a slice to
+	// collect all entries just to check for emptiness.
+	it := dir.Entries()
+	hasEntry := it.Next()
+	if !hasEntry {
+		if err := it.Err(); err != nil {
+			return err
+		}
+		// Directory is empty. Skip it unless IncludeEmptyDirs is set or
+		// this is the toplevel directory (we always include the root).
+		if !adder.IncludeEmptyDirs && !toplevel {
+			log.Debugf("skipping empty directory: %s", path)
+			return nil
+		}
+	}
+
 	// if we need to store mode or modification time then create a new root which includes that data
 	if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) {
 		mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil,
@@ -515,13 +536,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory
 		}
 	}
 
-	it := dir.Entries()
-	for it.Next() {
+	// Process directory entries. The iterator was already advanced once above
+	// to peek for emptiness, so we start from that position.
+	for hasEntry {
 		fpath := gopath.Join(path, it.Name())
-		err := adder.addFileNode(ctx, fpath, it.Node(), false)
-		if err != nil {
+		if err := adder.addFileNode(ctx, fpath, it.Node(), false); err != nil {
 			return err
 		}
+		hasEntry = it.Next()
 	}
 
 	return it.Err()