feat(add): add --dereference-symlinks, --empty-dirs, --hidden CLI flags

add CLI flags for controlling file collection behavior during ipfs add:

- `--dereference-symlinks`: recursively resolve symlinks to their target
  content (replaces deprecated --dereference-args which only worked on
  CLI arguments). wired through go-ipfs-cmds to boxo's SerialFileOptions.
- `--empty-dirs` / `-E`: include empty directories (default: true)
- `--hidden` / `-H`: include hidden files (default: false)

these flags are CLI-only and not wired to Import.* config options because
go-ipfs-cmds library handles input file filtering before the directory
tree is passed to kubo. removed unused Import.UnixFSSymlinkMode config
option that was defined but never actually read by the CLI.

also:
- wire --trickle to Import.UnixFSDAGLayout config default
- update go-ipfs-cmds to v0.15.1-0.20260117043932-17687e216294
- add SYMLINK HANDLING section to ipfs add help text
- add CLI tests for all three flags

ref: ipfs/specs#499

Author: lidel

config/import.go

@@ -35,17 +35,13 @@ const (
 	HAMTSizeEstimationBlock    = "block"    // full serialized dag-pb block size
 	HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely
 
-	// SymlinkMode values for Import.UnixFSSymlinkMode
-	SymlinkModePreserve    = "preserve"    // preserve symlinks as UnixFS symlink nodes
-	SymlinkModeDereference = "dereference" // dereference symlinks, import target content
-
 	// DAGLayout values for Import.UnixFSDAGLayout
 	DAGLayoutBalanced = "balanced" // balanced DAG layout (default)
 	DAGLayoutTrickle  = "trickle"  // trickle DAG layout
 
 	DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior
-	DefaultUnixFSSymlinkMode                 = SymlinkModePreserve     // preserve symlinks as UnixFS symlink nodes
 	DefaultUnixFSDAGLayout                   = DAGLayoutBalanced       // balanced DAG layout
+	DefaultUnixFSIncludeEmptyDirs            = true                    // include empty directories
 )
 
 var (
@@ -66,7 +62,6 @@ type Import struct {
 	UnixFSHAMTDirectoryMaxFanout      OptionalInteger
 	UnixFSHAMTDirectorySizeThreshold  OptionalBytes
 	UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled"
-	UnixFSSymlinkMode                 OptionalString // "preserve" or "dereference"
 	UnixFSDAGLayout                   OptionalString // "balanced" or "trickle"
 	BatchMaxNodes                     OptionalInteger
 	BatchMaxSize                      OptionalInteger
@@ -161,18 +156,6 @@ func ValidateImportConfig(cfg *Import) error {
 		}
 	}
 
-	// Validate UnixFSSymlinkMode
-	if !cfg.UnixFSSymlinkMode.IsDefault() {
-		mode := cfg.UnixFSSymlinkMode.WithDefault(DefaultUnixFSSymlinkMode)
-		switch mode {
-		case SymlinkModePreserve, SymlinkModeDereference:
-			// valid
-		default:
-			return fmt.Errorf("Import.UnixFSSymlinkMode must be %q or %q, got %q",
-				SymlinkModePreserve, SymlinkModeDereference, mode)
-		}
-	}
-
 	// Validate UnixFSDAGLayout
 	if !cfg.UnixFSDAGLayout.IsDefault() {
 		layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout)

config/import_test.go

@@ -446,43 +446,6 @@ func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) {
 	}
 }
 
-func TestValidateImportConfig_SymlinkMode(t *testing.T) {
-	tests := []struct {
-		name    string
-		value   string
-		wantErr bool
-		errMsg  string
-	}{
-		{name: "valid preserve", value: SymlinkModePreserve, wantErr: false},
-		{name: "valid dereference", value: SymlinkModeDereference, wantErr: false},
-		{name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"},
-		{name: "invalid empty", value: "", wantErr: true, errMsg: "must be"},
-		{name: "invalid follow", value: "follow", wantErr: true, errMsg: "must be"},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			cfg := &Import{
-				UnixFSSymlinkMode: *NewOptionalString(tt.value),
-			}
-
-			err := ValidateImportConfig(cfg)
-
-			if tt.wantErr {
-				if err == nil {
-					t.Errorf("expected error for value=%q, got nil", tt.value)
-				} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
-					t.Errorf("error = %v, want error containing %q", err, tt.errMsg)
-				}
-			} else {
-				if err != nil {
-					t.Errorf("unexpected error for value=%q: %v", tt.value, err)
-				}
-			}
-		})
-	}
-}
-
 func TestValidateImportConfig_DAGLayout(t *testing.T) {
 	tests := []struct {
 		name    string

config/profile.go

@@ -338,7 +338,6 @@ See https://github.com/ipfs/specs/pull/499`,
 			c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
 			c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB")
 			c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock)
-			c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve)
 			c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced)
 			return nil
 		},
@@ -436,7 +435,6 @@ func applyUnixFSv02015(c *Config) error {
 	c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
 	c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB")
 	c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks)
-	c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve)
 	c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced)
 	return nil
 }

core/commands/add.go

@@ -68,6 +68,7 @@ const (
 	mtimeNsecsOptionName      = "mtime-nsecs"
 	fastProvideRootOptionName = "fast-provide-root"
 	fastProvideWaitOptionName = "fast-provide-wait"
+	emptyDirsOptionName       = "empty-dirs"
 )
 
 const (
@@ -147,6 +148,18 @@ to find it in the future:
 See 'ipfs files --help' to learn more about using MFS
 for keeping track of added files and directories.
 
+SYMLINK HANDLING:
+
+By default, symbolic links are preserved as UnixFS symlink nodes that store
+the target path. Use --dereference-symlinks to resolve symlinks to their
+target content instead:
+
+  > ipfs add -r --dereference-symlinks ./mydir
+
+This recursively resolves all symlinks encountered during directory traversal.
+Symlinks to files become regular file content, symlinks to directories are
+traversed and their contents are added.
+
 CHUNKING EXAMPLES:
 
 The chunker option, '-s', specifies the chunking strategy that dictates
@@ -200,11 +213,13 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 	Options: []cmds.Option{
 		// Input Processing
 		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
-		cmds.OptionDerefArgs,     // a builtin option that resolves passed in filesystem links (--dereference-args)
+		cmds.OptionDerefArgs,     // DEPRECATED: use --dereference-symlinks instead
 		cmds.OptionStdinName,     // a builtin option that optionally allows wrapping stdin into a named file
 		cmds.OptionHidden,
 		cmds.OptionIgnore,
 		cmds.OptionIgnoreRules,
+		cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs),
+		cmds.OptionDerefSymlinks, // resolve symlinks to their target content
 		// Output Control
 		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
 		cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."),
@@ -274,7 +289,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 		}
 
 		progress, _ := req.Options[progressOptionName].(bool)
-		trickle, _ := req.Options[trickleOptionName].(bool)
+		trickle, trickleSet := req.Options[trickleOptionName].(bool)
 		wrap, _ := req.Options[wrapOptionName].(bool)
 		onlyHash, _ := req.Options[onlyHashOptionName].(bool)
 		silent, _ := req.Options[silentOptionName].(bool)
@@ -312,6 +327,19 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 		mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
 		fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
 		fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)
+		emptyDirs, _ := req.Options[emptyDirsOptionName].(bool)
+
+		// Handle --dereference-args deprecation
+		derefArgs, derefArgsSet := req.Options[cmds.DerefLong].(bool)
+		if derefArgsSet && derefArgs {
+			return fmt.Errorf("--dereference-args is deprecated: use --dereference-symlinks instead")
+		}
+
+		// Wire --trickle from config
+		if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() {
+			layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout)
+			trickle = layout == config.DAGLayoutTrickle
+		}
 
 		if chunker == "" {
 			chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
@@ -409,6 +437,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import
 
 			options.Unixfs.PreserveMode(preserveMode),
 			options.Unixfs.PreserveMtime(preserveMtime),
+
+			options.Unixfs.IncludeEmptyDirs(emptyDirs),
 		}
 
 		if mode != 0 {

core/coreapi/unixfs.go

@@ -183,6 +183,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	fileAdder.PreserveMtime = settings.PreserveMtime
 	fileAdder.FileMode = settings.Mode
 	fileAdder.FileMtime = settings.Mtime
+	if settings.IncludeEmptyDirsSet {
+		fileAdder.IncludeEmptyDirs = settings.IncludeEmptyDirs
+	}
 
 	switch settings.Layout {
 	case options.BalancedLayout:

core/coreiface/options/unixfs.go

@@ -48,10 +48,12 @@ type UnixfsAddSettings struct {
 	Silent   bool
 	Progress bool
 
-	PreserveMode  bool
-	PreserveMtime bool
-	Mode          os.FileMode
-	Mtime         time.Time
+	PreserveMode      bool
+	PreserveMtime     bool
+	Mode              os.FileMode
+	Mtime             time.Time
+	IncludeEmptyDirs  bool
+	IncludeEmptyDirsSet bool
 }
 
 type UnixfsLsSettings struct {
@@ -93,10 +95,12 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix,
 		Silent:   false,
 		Progress: false,
 
-		PreserveMode:  false,
-		PreserveMtime: false,
-		Mode:          0,
-		Mtime:         time.Time{},
+		PreserveMode:        false,
+		PreserveMtime:       false,
+		Mode:                0,
+		Mtime:               time.Time{},
+		IncludeEmptyDirs:    true, // default: include empty directories
+		IncludeEmptyDirsSet: false,
 	}
 
 	for _, opt := range opts {
@@ -396,3 +400,12 @@ func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption {
 		return nil
 	}
 }
+
+// IncludeEmptyDirs tells the adder to include empty directories in the DAG
+func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.IncludeEmptyDirs = include
+		settings.IncludeEmptyDirsSet = true
+		return nil
+	}
+}

core/coreunix/add.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/ipfs/kubo/config"
 	coreiface "github.com/ipfs/kubo/core/coreiface"
 
 	"github.com/ipfs/kubo/tracing"
@@ -52,17 +53,18 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG
 	bufferedDS := ipld.NewBufferedDAG(ctx, ds)
 
 	return &Adder{
-		ctx:           ctx,
-		pinning:       p,
-		gcLocker:      bs,
-		dagService:    ds,
-		bufferedDS:    bufferedDS,
-		Progress:      false,
-		Pin:           true,
-		Trickle:       false,
-		MaxLinks:      ihelper.DefaultLinksPerBlock,
-		MaxHAMTFanout: uio.DefaultShardWidth,
-		Chunker:       "",
+		ctx:              ctx,
+		pinning:          p,
+		gcLocker:         bs,
+		dagService:       ds,
+		bufferedDS:       bufferedDS,
+		Progress:         false,
+		Pin:              true,
+		Trickle:          false,
+		MaxLinks:         ihelper.DefaultLinksPerBlock,
+		MaxHAMTFanout:    uio.DefaultShardWidth,
+		Chunker:          "",
+		IncludeEmptyDirs: config.DefaultUnixFSIncludeEmptyDirs,
 	}, nil
 }
 
@@ -91,10 +93,11 @@ type Adder struct {
 	CidBuilder        cid.Builder
 	liveNodes         uint64
 
-	PreserveMode  bool
-	PreserveMtime bool
-	FileMode      os.FileMode
-	FileMtime     time.Time
+	PreserveMode     bool
+	PreserveMtime    bool
+	FileMode         os.FileMode
+	FileMtime        time.Time
+	IncludeEmptyDirs bool
 }
 
 func (adder *Adder) mfsRoot() (*mfs.Root, error) {
@@ -480,6 +483,24 @@ func (adder *Adder) addFile(path string, file files.File) error {
 func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error {
 	log.Infof("adding directory: %s", path)
 
+	// Peek at first entry to check if directory is empty.
+	// We advance the iterator once here and continue from this position
+	// in the processing loop below. This avoids allocating a slice to
+	// collect all entries just to check for emptiness.
+	it := dir.Entries()
+	hasEntry := it.Next()
+	if !hasEntry {
+		if err := it.Err(); err != nil {
+			return err
+		}
+		// Directory is empty. Skip it unless IncludeEmptyDirs is set or
+		// this is the toplevel directory (we always include the root).
+		if !adder.IncludeEmptyDirs && !toplevel {
+			log.Debugf("skipping empty directory: %s", path)
+			return nil
+		}
+	}
+
 	// if we need to store mode or modification time then create a new root which includes that data
 	if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) {
 		mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil,
@@ -515,13 +536,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory
 		}
 	}
 
-	it := dir.Entries()
-	for it.Next() {
+	// Process directory entries. The iterator was already advanced once above
+	// to peek for emptiness, so we start from that position.
+	for hasEntry {
 		fpath := gopath.Join(path, it.Name())
-		err := adder.addFileNode(ctx, fpath, it.Node(), false)
-		if err != nil {
+		if err := adder.addFileNode(ctx, fpath, it.Node(), false); err != nil {
 			return err
 		}
+		hasEntry = it.Next()
 	}
 
 	return it.Err()

docs/changelogs/v0.40.md

@@ -10,6 +10,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+  - [🔢 UnixFS CID Profiles (IPIP-499)](#-unixfs-cid-profiles-ipip-499)
   - [🧹 Automatic cleanup of interrupted imports](#-automatic-cleanup-of-interrupted-imports)
   - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default)
   - [Track total size when adding pins](#track-total-size-when-adding-pins)
@@ -30,6 +31,43 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
 
 ### 🔦 Highlights
 
+#### 🔢 UnixFS CID Profiles (IPIP-499)
+
+This release introduces [IPIP-499](https://github.com/ipfs/specs/pull/499) UnixFS CID Profiles for reproducible CID generation across IPFS implementations.
+
+**New Profiles**
+
+- `unixfs-v1-2025`: the recommended profile for CIDv1 imports with deterministic HAMT directory sharding based on block size estimation
+- `unixfs-v0-2015` (alias `legacy-cid-v0`): preserves legacy CIDv0 behavior for backward compatibility
+
+Apply a profile with: `ipfs config profile apply unixfs-v1-2025`
+
+**New `Import.*` Configuration Options**
+
+New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options allow fine-grained control over import parameters:
+
+- `Import.CidVersion`: CID version (0 or 1)
+- `Import.HashFunction`: hash algorithm
+- `Import.UnixFSChunker`: chunking strategy
+- `Import.UnixFSRawLeaves`: raw leaf blocks
+- `Import.UnixFSFileMaxLinks`: max children per file node
+- `Import.UnixFSDirectoryMaxLinks`: max children per basic directory
+- `Import.UnixFSHAMTDirectoryMaxFanout`: HAMT shard width
+- `Import.UnixFSHAMTDirectorySizeThreshold`: threshold for HAMT sharding
+- `Import.UnixFSHAMTDirectorySizeEstimation`: estimation mode (`links`, `block`, or `disabled`)
+- `Import.UnixFSDAGLayout`: DAG layout (`balanced` or `trickle`)
+
+**Deprecated Profiles**
+
+The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` for CIDv1 imports.
+
+**CLI Changes**
+
+- New `--dereference-symlinks` flag for `ipfs add` recursively resolves symlinks to their target content (replaces deprecated `--dereference-args` which only worked on CLI arguments)
+- New `--empty-dirs` / `-E` flag for `ipfs add` controls inclusion of empty directories (default: true)
+- New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false)
+- The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default
+
 #### 🧹 Automatic cleanup of interrupted imports
 
 If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection.