diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 97462d60ea6..176f34e807d 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -212,6 +212,17 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { } t.Logf("config B loaded from %s: %+v", testFile, string(configBPretty)) + // Config A written from uncommented file with scm_hugepages_disabled: false + // set explicitly whereas when config B yaml gets written, + // scm_hugepages_disabled is omitted (omitempty) and therefore when read is + // set to true by default. Adjust configB because of this. + if len(configB.Engines) == 2 { + for _, e := range configB.Engines { + ramCfg := e.Storage.Tiers.ScmConfigs()[0] + ramCfg.Scm.DisableHugepages = false + } + } + if diff := cmp.Diff(configA, configB, defConfigCmpOpts...); diff != "" { t.Fatalf("(-want, +got): %s", diff) } @@ -279,7 +290,7 @@ func TestServerConfig_Constructed(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(false), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -309,7 +320,7 @@ func TestServerConfig_Constructed(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/2"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(false), storage.NewTierConfig(). WithStorageClass("file"). WithBdevDeviceList("/tmp/daos-bdev1", "/tmp/daos-bdev2"). @@ -443,7 +454,8 @@ func TestServerConfig_MDonSSD_Constructed(t *testing.T) { WithStorage( storage.NewTierConfig(). WithScmMountPoint("/mnt/daos"). - WithStorageClass("ram"), + WithStorageClass("ram"). + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0"). @@ -698,7 +710,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -711,7 +723,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/2"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:91:00.0", "0000:92:00.0"). @@ -733,7 +745,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -756,7 +768,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/2"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:91:00.0", "0000:92:00.0"). @@ -787,7 +799,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -808,7 +820,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -838,7 +850,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/0"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:80:00.0"). @@ -850,7 +862,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0"), @@ -871,7 +883,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -890,7 +902,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -934,7 +946,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0"), @@ -953,7 +965,7 @@ func TestServerConfig_Validation(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0"). @@ -1139,7 +1151,7 @@ func TestServerConfig_getMinNrHugepages(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -1275,7 +1287,7 @@ func TestServerConfig_SetNrHugepages(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -1295,7 +1307,7 @@ func TestServerConfig_SetNrHugepages(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -1479,7 +1491,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { storage.NewTierConfig(). WithScmMountPoint("/mnt/daos/1"). WithStorageClass("ram"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("nvme"). WithBdevDeviceList("0000:81:00.0", "0000:82:00.0"). @@ -1735,12 +1747,24 @@ func TestServerConfig_Parsing(t *testing.T) { return nil }, }, - "allow_thp true allows scm_hugepages_disabled false": { - inTxt: " scm_hugepages_disabled: true", - outTxt: " scm_hugepages_disabled: false", + "implicitly set scm_hugepages_disabled true": { + inTxt: " scm_hugepages_disabled: false", + outTxt: "", + expCheck: func(c *Server) error { + for _, e := range c.Engines { + if !e.Storage.Tiers.ScmConfigs()[0].Scm.DisableHugepages { + return errors.New("expecting scm hugepages to be enabled") + } + } + return nil + }, + }, + "explicitly set scm_hugepages_disabled true": { + inTxt: " scm_hugepages_disabled: false", + outTxt: " scm_hugepages_disabled: true", expCheck: func(c *Server) error { for _, e := range c.Engines { - if e.Storage.Tiers.ScmConfigs()[0].Scm.DisableHugepages { + if !e.Storage.Tiers.ScmConfigs()[0].Scm.DisableHugepages { return errors.New("expecting scm hugepages to be enabled") } } @@ -2012,7 +2036,7 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { storage.NewTierConfig(). WithStorageClass("ram"). WithScmMountPoint("b"). - WithScmHugepagesDisabled(), + WithScmHugepagesDisabled(true), ), expErr: FaultConfigScmDiffHugeEnabled(1, 0), }, diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 48c29a7b794..5811e06d6ef 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -140,7 +140,8 @@ func TestConfig_Constructed(t *testing.T) { WithStorageClass("ram"). WithScmRamdiskSize(42). WithScmMountPoint("/mnt/daostest"). - WithScmDeviceList("/dev/a", "/dev/b"), + WithScmDeviceList("/dev/a", "/dev/b"). + WithScmHugepagesDisabled(true), storage.NewTierConfig(). WithStorageClass("kdev"). WithBdevDeviceCount(2). diff --git a/src/control/server/storage/config.go b/src/control/server/storage/config.go index 548c959aee7..ab72088722f 100644 --- a/src/control/server/storage/config.go +++ b/src/control/server/storage/config.go @@ -174,8 +174,8 @@ func (tc *TierConfig) WithStorageClass(cls string) *TierConfig { } // WithScmHugepagesDisabled disables hugepages for tmpfs. -func (tc *TierConfig) WithScmHugepagesDisabled() *TierConfig { - tc.Scm.DisableHugepages = true +func (tc *TierConfig) WithScmHugepagesDisabled(b bool) *TierConfig { + tc.Scm.DisableHugepages = b return tc } @@ -241,6 +241,26 @@ func (tc *TierConfig) WithNumaNodeIndex(idx uint) *TierConfig { return tc } +// UnmarshalYAML sets SCM DisableHugepages to true unless configured otherwise. +func (tc *TierConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults before unmarshaling + type Alias TierConfig + defaults := &Alias{} + defaults.Scm.DisableHugepages = true // Default to disabling hugepages + + if err := unmarshal(defaults); err != nil { + return err + } + + // For classes other than RAM, reset to zero value as setting is irrelevant. + if defaults.Class != ClassRam { + defaults.Scm.DisableHugepages = false + } + + *tc = TierConfig(*defaults) + return nil +} + type TierConfigs []*TierConfig func (tcs TierConfigs) getBdevs(nvmeOnly bool) *BdevDeviceList { @@ -593,9 +613,6 @@ func (sc *ScmConfig) Validate(class Class) error { if len(sc.DeviceList) == 0 { return errors.New("scm_list must be set when class is dcpm") } - if sc.DisableHugepages { - return errors.New("scm_hugepages_disabled may not be set when class is dcpm") - } case ClassRam: if len(sc.DeviceList) > 0 { return errors.New("scm_list may not be set when class is ram") diff --git a/src/control/server/storage/config_test.go b/src/control/server/storage/config_test.go index e3dc3053278..869932211c7 100644 --- a/src/control/server/storage/config_test.go +++ b/src/control/server/storage/config_test.go @@ -420,7 +420,8 @@ storage: NewTierConfig(). WithStorageClass("ram"). WithScmRamdiskSize(16). - WithScmMountPoint("/mnt/daos"), + WithScmMountPoint("/mnt/daos"). + WithScmHugepagesDisabled(true), NewTierConfig(). WithTier(1). WithStorageClass("nvme"). @@ -527,7 +528,8 @@ storage: NewTierConfig(). WithStorageClass("ram"). WithScmRamdiskSize(16). - WithScmMountPoint("/mnt/daos"), + WithScmMountPoint("/mnt/daos"). + WithScmHugepagesDisabled(true), NewTierConfig(). WithTier(1). WithStorageClass("nvme"). @@ -565,7 +567,8 @@ storage: NewTierConfig(). WithStorageClass("ram"). WithScmRamdiskSize(16). - WithScmMountPoint("/mnt/daos"), + WithScmMountPoint("/mnt/daos"). + WithScmHugepagesDisabled(true), NewTierConfig(). WithTier(1). WithStorageClass("nvme"). @@ -1213,8 +1216,9 @@ storage: &TierConfig{ Class: ClassRam, Scm: ScmConfig{ - MountPoint: "/mnt/daos/1", - RamdiskSize: 16, + MountPoint: "/mnt/daos/1", + RamdiskSize: 16, + DisableHugepages: true, }, }, }, @@ -1234,8 +1238,9 @@ storage: &TierConfig{ Class: ClassRam, Scm: ScmConfig{ - MountPoint: "/mnt/daos/1", - RamdiskSize: 16, + MountPoint: "/mnt/daos/1", + RamdiskSize: 16, + DisableHugepages: true, }, }, &TierConfig{ @@ -1259,8 +1264,9 @@ storage: &TierConfig{ Class: ClassRam, Scm: ScmConfig{ - MountPoint: "/mnt/daos/1", - RamdiskSize: 16, + MountPoint: "/mnt/daos/1", + RamdiskSize: 16, + DisableHugepages: true, }, }, &TierConfig{ @@ -1284,8 +1290,9 @@ storage: &TierConfig{ Class: ClassRam, Scm: ScmConfig{ - MountPoint: "/mnt/daos/1", - RamdiskSize: 16, + MountPoint: "/mnt/daos/1", + RamdiskSize: 16, + DisableHugepages: true, }, }, &TierConfig{ @@ -1310,8 +1317,9 @@ storage: &TierConfig{ Class: ClassRam, Scm: ScmConfig{ - MountPoint: "/mnt/daos/1", - RamdiskSize: 16, + MountPoint: "/mnt/daos/1", + RamdiskSize: 16, + DisableHugepages: true, }, }, &TierConfig{ @@ -1471,3 +1479,33 @@ func TestStorage_Config_Validate(t *testing.T) { }) } } + +func TestStorage_ScmConfig_DisableHugepages_MultipleEngines(t *testing.T) { + yamlInput := ` +- class: ram + scm_mount: /mnt/daos0 + scm_size: 16 +- class: ram + scm_mount: /mnt/daos1 + scm_size: 16 + scm_hugepages_disabled: false +` + var configs TierConfigs + if err := yaml.Unmarshal([]byte(yamlInput), &configs); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + if len(configs) != 2 { + t.Fatalf("expected 2 configs, got %d", len(configs)) + } + + // First config should have DisableHugepages=true (default) + if !configs[0].Scm.DisableHugepages { + t.Fatalf("expected engine 0 DisableHugepages=true, got false") + } + + // Second config explicitly set to false + if configs[1].Scm.DisableHugepages { + t.Fatalf("expected engine 1 DisableHugepages=false, got true") + } +} diff --git a/src/tests/ftest/pool/create.yaml b/src/tests/ftest/pool/create.yaml index c3e06b86211..87a5a385fab 100644 --- a/src/tests/ftest/pool/create.yaml +++ b/src/tests/ftest/pool/create.yaml @@ -6,7 +6,7 @@ timeouts: test_create_max_pool_scm_only: 180 test_create_max_pool: 300 test_create_no_space: 300 - test_create_no_space_loop: 3500 + test_create_no_space_loop: 3700 server_config: name: daos_server diff --git a/utils/config/daos_server.yml b/utils/config/daos_server.yml index 88e93fcd3f1..352c85d87e2 100644 --- a/utils/config/daos_server.yml +++ b/utils/config/daos_server.yml @@ -417,10 +417,10 @@ # # #scm_size: 0 # -# # When class is set to ram, tmpfs will be mounted with hugepage -# # support, if the kernel supports it. If this is not desirable, -# # the behavior may be disabled here. -# scm_hugepages_disabled: true +# # When class is set to ram, tmpfs will only be mounted with hugepage +# # support if the kernel supports it and this flag is explicitly set to false. +# # default: true +# scm_hugepages_disabled: false # # - # # Backend block device type. Force a SPDK driver to be used by this engine @@ -589,10 +589,10 @@ # #class: dcpm # #scm_list: [/dev/pmem1] # -# # When class is set to ram, tmpfs will be mounted with hugepage -# # support, if the kernel supports it. If this is not desirable, -# # the behavior may be disabled here. -# scm_hugepages_disabled: true +# # When class is set to ram, tmpfs will only be mounted with hugepage +# # support if the kernel supports it and this flag is explicitly set to false. +# # default: true +# scm_hugepages_disabled: false # # - # # Backend block device type. Force a SPDK driver to be used by this engine