diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index e49214ca111..82f122b11c5 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -48,8 +48,18 @@ type FabricConfig struct { NumSecondaryEndpoints []int `yaml:"secondary_provider_endpoints,omitempty" cmdLongFlag:"--nr_sec_ctx,nonzero" cmdShortFlag:"-S,nonzero"` DisableSRX bool `yaml:"disable_srx,omitempty" cmdEnv:"FI_OFI_RXM_USE_SRX,invertBool,intBool"` AuthKey string `yaml:"fabric_auth_key,omitempty" cmdEnv:"D_PROVIDER_AUTH_KEY"` + // AddrFormat selects the preferred IP address family for fabric init. It is + // forwarded verbatim to CaRT/Mercury via the D_ADDR_FORMAT environment + // variable. An empty value leaves the historical (IPv4-preferring) default + // untouched. For multi-provider configurations it accepts a comma-separated + // list, one entry per provider, matching the provider ordering. + AddrFormat string `yaml:"addr_format,omitempty" cmdEnv:"D_ADDR_FORMAT"` } +// FabricAddrFormats enumerates the address-family hints accepted by addr_format, +// mirroring the values parsed by CaRT's crt_str_to_addr_format(). +var FabricAddrFormats = []string{"unspec", "ipv4", "ipv6", "native"} + // GetPrimaryProvider parses the primary provider from the Provider string. func (fc *FabricConfig) GetPrimaryProvider() (string, error) { providers, err := fc.GetProviders() @@ -120,6 +130,17 @@ func (fc *FabricConfig) GetInterfaces() ([]string, error) { return interfaces, nil } +// GetAddrFormats parses the AddrFormat string into zero or more per-provider +// address-family hints, matched one-to-one with the providers. Returns an empty +// slice when addr_format is unset, which preserves the default behavior. +func (fc *FabricConfig) GetAddrFormats() []string { + if fc == nil { + return nil + } + + return splitMultiProviderStr(fc.AddrFormat) +} + // GetInterfacePorts parses the InterfacePort string to one or more ports. func (fc *FabricConfig) GetInterfacePorts() ([]int, error) { if fc == nil { @@ -157,6 +178,9 @@ func (fc *FabricConfig) Update(other FabricConfig) { if fc.AuthKey == "" { fc.AuthKey = other.AuthKey } + if fc.AddrFormat == "" { + fc.AddrFormat = other.AddrFormat + } if len(fc.NumSecondaryEndpoints) == 0 { fc.setNumSecondaryEndpoints(other.NumSecondaryEndpoints) } @@ -200,6 +224,10 @@ func (fc *FabricConfig) Validate() error { return errors.Errorf("provider, fabric_iface and fabric_iface_port must include the same number of items delimited by %q", MultiProviderSeparator) } + if err := fc.validateAddrFormat(numProv); err != nil { + return err + } + numSecProv := numProv - numPrimaryProviders if numSecProv > 0 { if len(fc.NumSecondaryEndpoints) != 0 && len(fc.NumSecondaryEndpoints) != numSecProv { @@ -216,6 +244,31 @@ func (fc *FabricConfig) Validate() error { return nil } +// validateAddrFormat ensures every comma-separated addr_format entry is a +// recognized address-family hint and that the count matches the number of +// providers (one hint per provider, as with fabric_iface). An empty +// addr_format is valid and preserves the default behavior. +func (fc *FabricConfig) validateAddrFormat(numProv int) error { + if fc.AddrFormat == "" { + return nil + } + + formats := splitMultiProviderStr(fc.AddrFormat) + for _, f := range formats { + if !common.Includes(FabricAddrFormats, f) { + return errors.Errorf("invalid addr_format %q, must be one of %v", + f, FabricAddrFormats) + } + } + + if len(formats) != numProv { + return errors.Errorf("addr_format must include one value per provider, delimited by %q", + MultiProviderSeparator) + } + + return nil +} + // cleanEnvVars scrubs the supplied slice of environment // variables by removing all variables not included in the // allow list. @@ -711,6 +764,14 @@ func (c *Config) WithFabricAuthKey(key string) *Config { return c } +// WithFabricAddrFormat sets the preferred IP address family for fabric init +// (forwarded to CaRT/Mercury as D_ADDR_FORMAT). Accepted values are listed in +// FabricAddrFormats; an empty value preserves the default behavior. +func (c *Config) WithFabricAddrFormat(format string) *Config { + c.Fabric.AddrFormat = format + return c +} + // WithSrxDisabled disables or enables SRX. func (c *Config) WithSrxDisabled(disable bool) *Config { c.Fabric.DisableSRX = disable diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 1c771cfe8b5..c92139f07ad 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -693,6 +693,7 @@ func TestConfig_ToCmdVals(t *testing.T) { WithFabricProvider(provider). WithFabricInterface(interfaceName). WithFabricInterfacePort(interfacePort). + WithFabricAddrFormat("ipv6"). WithPinnedNumaNode(pinnedNumaNode). WithBypassHealthChk(&bypass). WithModules(modules). @@ -727,6 +728,7 @@ func TestConfig_ToCmdVals(t *testing.T) { "D_INTERFACE=" + interfaceName, "D_PORT=" + strconv.Itoa(interfacePort), "D_PROVIDER=" + provider, + "D_ADDR_FORMAT=ipv6", "D_LOG_FILE=" + logFile, "D_LOG_MASK=" + logMask, "CRT_TIMEOUT=" + strconv.FormatUint(uint64(crtTimeout), 10), @@ -899,6 +901,47 @@ func TestFabricConfig_GetInterfaces(t *testing.T) { } } +func TestFabricConfig_GetAddrFormats(t *testing.T) { + for name, tc := range map[string]struct { + cfg *FabricConfig + expAddrFormats []string + }{ + "nil": { + expAddrFormats: nil, + }, + "unset": { + cfg: &FabricConfig{}, + expAddrFormats: []string{}, + }, + "single": { + cfg: &FabricConfig{ + AddrFormat: "ipv6", + }, + expAddrFormats: []string{"ipv6"}, + }, + "multi": { + cfg: &FabricConfig{ + AddrFormat: multiProviderString("ipv6", "ipv4"), + }, + expAddrFormats: []string{"ipv6", "ipv4"}, + }, + "excessive whitespace": { + cfg: &FabricConfig{ + AddrFormat: multiProviderString(" ipv6 ", "", "ipv4 "), + }, + expAddrFormats: []string{"ipv6", "ipv4"}, + }, + } { + t.Run(name, func(t *testing.T) { + addrFormats := tc.cfg.GetAddrFormats() + + if diff := cmp.Diff(tc.expAddrFormats, addrFormats); diff != "" { + t.Fatalf("(-want, +got):\n%s", diff) + } + }) + } +} + func TestFabricConfig_GetPrimaryInterface(t *testing.T) { for name, tc := range map[string]struct { cfg *FabricConfig diff --git a/src/control/server/server.go b/src/control/server/server.go index 69fa0703a4d..fafb846ebca 100644 --- a/src/control/server/server.go +++ b/src/control/server/server.go @@ -448,15 +448,31 @@ func (srv *server) setupGrpc() error { return err } + // Address family is a property of this system's fabric, so it must travel + // to attaching clients alongside the provider. Surface the operator's + // addr_format choice as a D_ADDR_FORMAT entry in the per-provider client + // hint env, which the client applies before crt_init (see + // dc_mgmt_net_cfg_init). A client/agent attached to several systems gets + // each system's family from that system's own hint. Empty addr_format + // leaves the hint untouched, preserving the historical default. + addrFormats := srv.cfg.Fabric.GetAddrFormats() + clientNetHints := make([]*mgmtpb.ClientNetHint, 0, len(providers)) for i, p := range providers { + envVars := srv.cfg.ClientEnvVars + if i < len(addrFormats) && addrFormats[i] != "" { + // Copy rather than append in place to avoid aliasing the shared + // ClientEnvVars slice across providers. + envVars = append(append([]string{}, envVars...), "D_ADDR_FORMAT="+addrFormats[i]) + } + clientNetHints = append(clientNetHints, &mgmtpb.ClientNetHint{ Provider: p, CrtTimeout: srv.cfg.Fabric.CrtTimeout, NetDevClass: uint32(srv.netDevClass[i]), SrvSrxSet: srxSetting, ProviderIdx: uint32(i), - EnvVars: srv.cfg.ClientEnvVars, + EnvVars: envVars, }) } srv.mgmtSvc.clientNetworkHint = clientNetHints diff --git a/utils/config/daos_server.yml b/utils/config/daos_server.yml index a16693c3564..0bbd8d610c3 100644 --- a/utils/config/daos_server.yml +++ b/utils/config/daos_server.yml @@ -136,6 +136,24 @@ #fabric_auth_key: foo:bar # # +## CART: Fabric address family +## Selects the preferred IP address family for fabric initialization, +## forwarded to CaRT/Mercury as D_ADDR_FORMAT. Leave unset to keep the +## historical default (the provider's own preference; IPv4 for verbs/RoCE). +## Set to "ipv6" for IPv6-only fabric NIC deployments, where the default +## IPv4 preference would otherwise hide the only usable interfaces. +## Accepted values: unspec (default), ipv4, ipv6, native. For multi-provider +## configurations, supply a comma-separated list, one entry per provider. +## +## This value is also advertised to attaching clients (as D_ADDR_FORMAT in the +## GetAttachInfo network hint), so clients select the same address family as +## this system's engines without separate client configuration. A client or +## agent attached to several systems picks up each system's family from that +## system's hint. +# +#addr_format: ipv6 +# +# ## Core Dump Filter ## Optional filter to control which mappings are written to the core ## dump in the event of a crash. See the following URL for more detail: