diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 56191e68d9ebd..17d5bac31c914 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -83,6 +83,7 @@ import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.CollectionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; @@ -266,6 +267,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_AWARE_QUERIES_ENABLED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_SERIALIZABLE_ENABLED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_WAL_MODE; import static org.apache.ignite.internal.IgniteVersionUtils.BUILD_TSTAMP_STR; import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT; import static org.apache.ignite.internal.IgniteVersionUtils.VER; @@ -1651,8 +1653,12 @@ private void addDataStorageConfigurationAttributes() throws IgniteCheckedExcepti add(ATTR_MEMORY_CONFIG, memCfg); } + DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration(); + // Save data storage configuration. - add(ATTR_DATA_STORAGE_CONFIG, ctx.marshallerContext().jdkMarshaller().marshal(cfg.getDataStorageConfiguration())); + add(ATTR_DATA_STORAGE_CONFIG, ctx.marshallerContext().jdkMarshaller().marshal(dsCfg)); + + ctx.addNodeAttribute(ATTR_WAL_MODE, dsCfg == null ? null : dsCfg.getWalMode().ordinal()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index a5fc9270d40b7..7ca3c8b030fac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -202,6 +202,9 @@ public final class IgniteNodeAttributes { /** Data center ID. */ public static final String ATTR_DATA_CENTER_ID = ATTR_PREFIX + ".datacenter.id"; + /** WAL mode configuration. */ + public static final String ATTR_WAL_MODE = ATTR_PREFIX + ".wal.mode"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 7334cfc16c9a9..df3e6639efc03 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -58,6 +58,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.DefaultCommunicationFailureResolver; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.Event; import org.apache.ignite.events.EventType; @@ -176,6 +177,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_COMPATIBILITY_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SHUTDOWN_POLICY; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_WAL_MODE; import static org.apache.ignite.internal.IgniteVersionUtils.VER; import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; @@ -1279,6 +1281,9 @@ private void checkAttributes(Iterable nodes) throws IgniteCheckedEx Boolean locSecurityCompatibilityEnabled = locNode.attribute(ATTR_SECURITY_COMPATIBILITY_MODE); + WALMode locWalMode = locNode.attribute(ATTR_WAL_MODE) == null ? null : + WALMode.fromOrdinal(locNode.attribute(ATTR_WAL_MODE)); + for (ClusterNode n : nodes) { int rmtJvmMajVer = nodeJavaMajorVersion(n); @@ -1383,6 +1388,16 @@ private void checkAttributes(Iterable nodes) throws IgniteCheckedEx ", locNodeId=" + locNode.id() + ", rmtNode=" + U.toShortString(n) + "]"); } } + + WALMode rmtWalMode = n.attribute(ATTR_WAL_MODE) == null ? null : + WALMode.fromOrdinal(n.attribute(ATTR_WAL_MODE)); + + if (locWalMode != null && rmtWalMode != null && locWalMode != rmtWalMode) { + throw new IgniteCheckedException("Remote node has WAL mode different from local " + + "[locId8=" + U.id8(locNode.id()) + ", locWalMode=" + locWalMode + + ", rmtId8=" + U.id8(n.id()) + ", rmtWalMode=" + rmtWalMode + + ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]"); + } } if (log.isDebugEnabled()) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerWalModeConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerWalModeConsistencyTest.java new file mode 100644 index 0000000000000..f31763d65c5a8 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerWalModeConsistencyTest.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.discovery; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** Tests for WAL mode consistency validation when nodes join cluster. */ +public class GridDiscoveryManagerWalModeConsistencyTest extends GridCommonAbstractTest { + /** */ + private static final String PERSISTENT_REGION_NAME = "pds-reg"; + + /** */ + private WALMode walMode; + + /** */ + private boolean mixedConfig; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration storageCfg = new DataStorageConfiguration(); + + storageCfg.setWalMode(walMode); + + if (mixedConfig) { + storageCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setName("default_in_memory_region") + .setPersistenceEnabled(false)); + + if (igniteInstanceName.contains("persistent_instance")) { + DataRegionConfiguration persistentRegionCfg = new DataRegionConfiguration().setName(PERSISTENT_REGION_NAME) + .setPersistenceEnabled(true); + + storageCfg.setDataRegionConfigurations(persistentRegionCfg); + } + } + else { + storageCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(walMode != null); + } + + cfg.setDataStorageConfiguration(storageCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + walMode = null; + mixedConfig = false; + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** + * Tests that nodes with same WAL mode can join cluster successfully. + * + * @throws Exception If failed. + */ + @Test + public void testSameWalModeJoinsSuccessfully() throws Exception { + walMode = WALMode.LOG_ONLY; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx ignite1 = startGrid(1); + + IgniteEx cli = startClientGrid(2); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + assertEquals(3, ignite0.cluster().nodes().size()); + assertEquals(3, ignite1.cluster().nodes().size()); + } + + /** + * Tests that nodes with different WAL modes cannot join cluster. + * + * @throws Exception If failed. + */ + @Test + public void testDifferentWalModesCannotJoin() throws Exception { + walMode = WALMode.LOG_ONLY; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx cli = startClientGrid(2); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = WALMode.FSYNC; + + String errMsg = GridTestUtils.assertThrowsWithCause(() -> startGrid(1), IgniteCheckedException.class) + .getCause().getMessage(); + + assertTrue(errMsg.startsWith("Remote node has WAL mode different from local") && + (errMsg.contains("locWalMode=FSYNC") && errMsg.contains("rmtWalMode=LOG_ONLY")) || + (errMsg.contains("locWalMode=LOG_ONLY") && errMsg.contains("rmtWalMode=FSYNC"))); + + assertEquals(2, ignite0.cluster().nodes().size()); + } + + /** + * Tests that in-memory node can join cluster with persistence node with default WAL mode. + * + * @throws Exception If failed. + */ + @Test + public void testInMemoryNodeJoinsPersistenceNodeWithDefaultWalMode() throws Exception { + walMode = DataStorageConfiguration.DFLT_WAL_MODE; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx cli = startClientGrid(1); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = null; + GridTestUtils.assertThrowsWithCause(() -> startGrid(2), IgniteCheckedException.class); + + assertEquals(2, ignite0.cluster().nodes().size()); + } + + /** + * Tests that in-memory node can join cluster with persistence node with FSYNC WAL mode. + * + * @throws Exception If failed. + */ + @Test + public void testInMemoryNodeJoinsPersistenceNodeWithFsyncWalMode() throws Exception { + walMode = WALMode.FSYNC; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx cli = startClientGrid(1); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = null; + GridTestUtils.assertThrowsWithCause(() -> startGrid(2), IgniteCheckedException.class); + + assertEquals(2, ignite0.cluster().nodes().size()); + } + + /** + * Tests that persistence node with default WAL mode can join cluster. + * + * @throws Exception If failed. + */ + @Test + public void testPersistenceNodeWithDefaultWalModeJoinsSuccessfully() throws Exception { + walMode = DataStorageConfiguration.DFLT_WAL_MODE; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx cli = startClientGrid(2); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = DataStorageConfiguration.DFLT_WAL_MODE; + IgniteEx ignite1 = startGrid(1); + + assertEquals(3, ignite0.cluster().nodes().size()); + assertEquals(3, ignite1.cluster().nodes().size()); + } + + /** + * Tests that two in-memory nodes can join cluster successfully. + * + * @throws Exception If failed. + */ + @Test + public void testInMemoryNodesJoinSuccessfully() throws Exception { + walMode = null; + + IgniteEx ignite0 = startGrid(0); + + IgniteEx cli = startClientGrid(2); + + ignite0.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = null; + IgniteEx ignite1 = startGrid(1); + + assertEquals(3, ignite0.cluster().nodes().size()); + assertEquals(3, ignite1.cluster().nodes().size()); + } + + /** + * Tests mixed cluster: cluster is in-memory with null walMode, node with + * in-memory + persistent regions joins succesfully, walModes are the same + * + * @throws Exception If failed. + */ + @Test + public void testMixedClusterInMemoryJoinsPersistentWithDefaultWalMode() throws Exception { + mixedConfig = true; + walMode = null; + + IgniteEx inMemoryNode = startGrid("in-memory_instance"); + + IgniteEx cli = startClientGrid("client"); + + inMemoryNode.cluster().baselineAutoAdjustEnabled(false); + + inMemoryNode.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + startGrid("persistent_instance"); + + assertEquals(3, inMemoryNode.cluster().nodes().size()); + } + + /** + * Tests mixed cluster: in-memory node cluster, persistent node joins + * (has default in-memory region + persistent region with FSYNC WAL mode). + * + * @throws Exception If failed. + */ + @Test + public void testMixedClusterInMemoryJoinsPersistentWithFsyncWalMode() throws Exception { + mixedConfig = true; + walMode = null; + + IgniteEx inMemoryNode = startGrid("in-memory_instance"); + + IgniteEx cli = startClientGrid("client"); + + inMemoryNode.cluster().baselineAutoAdjustEnabled(false); + + inMemoryNode.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = WALMode.FSYNC; + String errMsg = GridTestUtils.assertThrowsWithCause(() -> startGrid("persistent_instance"), IgniteCheckedException.class) + .getCause().getMessage(); + + assertTrue(errMsg.startsWith("Remote node has WAL mode different from local") && + (errMsg.contains("locWalMode=FSYNC") && errMsg.contains("rmtWalMode=LOG_ONLY")) || + (errMsg.contains("locWalMode=LOG_ONLY") && errMsg.contains("rmtWalMode=FSYNC"))); + + assertEquals(2, inMemoryNode.cluster().nodes().size()); + } + + /** + * Tests mixed cluster: in-memory node cluster, persistent node joins + * (has default in-memory region + persistent region with Default WAL mode). + * + * @throws Exception If failed. + */ + @Test + public void testMixedClusterInMemoryJoinsPersistentWithDfltWalMode() throws Exception { + mixedConfig = true; + walMode = null; + + IgniteEx inMemoryNode = startGrid("in-memory_instance"); + + IgniteEx cli = startClientGrid("client"); + + inMemoryNode.cluster().baselineAutoAdjustEnabled(false); + + inMemoryNode.cluster().state(ClusterState.ACTIVE); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = DataStorageConfiguration.DFLT_WAL_MODE; + startGrid("persistent_instance"); + + assertEquals(3, inMemoryNode.cluster().nodes().size()); + } + + /** + * Tests mixed cluster: persistent node with default WAL mode cannot join + * cluster with persistent node with FSYNC WAL mode. + * + * @throws Exception If failed. + */ + @Test + public void testMixedClusterDifferentWalModesCannotJoin() throws Exception { + walMode = WALMode.FSYNC; + + IgniteEx persistentNode0 = startGrid("persistent_instance0"); + + persistentNode0.cluster().state(ClusterState.ACTIVE); + + IgniteEx cli = startClientGrid("client"); + + IgniteCache cacheCli = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cacheCli.put(1, 1); + + walMode = DataStorageConfiguration.DFLT_WAL_MODE; + + String errMsg = GridTestUtils.assertThrowsWithCause(() -> startGrid("persistent_instance1"), IgniteCheckedException.class) + .getCause().getMessage(); + + assertTrue(errMsg.startsWith("Remote node has WAL mode different from local") && + (errMsg.contains("locWalMode=FSYNC") && errMsg.contains("rmtWalMode=" + DataStorageConfiguration.DFLT_WAL_MODE.name())) || + (errMsg.contains("locWalMode=" + DataStorageConfiguration.DFLT_WAL_MODE.name()) && errMsg.contains("rmtWalMode=FSYNC"))); + + assertEquals(2, persistentNode0.cluster().nodes().size()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java index 90675f4aa0a02..dd6dd06042cb6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.managers.deployment.GridDeploymentManagerStopSelfTest; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAliveCacheSelfTest; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAttributesSelfTest; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerWalModeConsistencyTest; import org.apache.ignite.internal.managers.discovery.IgniteTopologyPrintFormatSelfTest; import org.apache.ignite.internal.managers.events.GridEventStorageManagerInternalEventsSelfTest; import org.apache.ignite.internal.managers.events.GridEventStorageManagerSelfTest; @@ -91,6 +92,7 @@ GridManagerStopSelfTest.class, GridDiscoveryManagerAttributesSelfTest.class, GridDiscoveryManagerAliveCacheSelfTest.class, + GridDiscoveryManagerWalModeConsistencyTest.class, GridDiscoveryEventSelfTest.class, GridPortProcessorSelfTest.class, GridHomePathSelfTest.class,