Skip to content

Commit

Permalink
ARTEMIS-2716 Support replicated failover tests
Browse files Browse the repository at this point in the history
  • Loading branch information
franz1981 committed May 4, 2021
1 parent 3a90d5d commit d2d53c6
Show file tree
Hide file tree
Showing 13 changed files with 767 additions and 6 deletions.
Expand Up @@ -130,6 +130,7 @@
import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl;
import org.apache.activemq.artemis.core.server.impl.AddressInfo;
import org.apache.activemq.artemis.core.server.impl.LiveOnlyActivation;
import org.apache.activemq.artemis.core.server.impl.ReplicationBackupActivation;
import org.apache.activemq.artemis.core.server.impl.SharedNothingBackupActivation;
import org.apache.activemq.artemis.core.settings.impl.AddressFullMessagePolicy;
import org.apache.activemq.artemis.core.settings.impl.AddressSettings;
Expand Down Expand Up @@ -1385,6 +1386,8 @@ public static final void waitForRemoteBackup(ClientSessionFactory sessionFactory
if (isReplicated) {
if (activation instanceof SharedNothingBackupActivation) {
isRemoteUpToDate = backup.isReplicaSync();
} else if (activation instanceof ReplicationBackupActivation) {
isRemoteUpToDate = backup.isReplicaSync();
} else {
//we may have already failed over and changed the Activation
if (actualServer.isStarted()) {
Expand Down Expand Up @@ -2523,6 +2526,9 @@ protected static ReplicationEndpoint getReplicationEndpoint(ActiveMQServer serve
if (activation instanceof SharedNothingBackupActivation) {
return ((SharedNothingBackupActivation) activation).getReplicationEndpoint();
}
if (activation instanceof ReplicationBackupActivation) {
return ((ReplicationBackupActivation) activation).getReplicationEndpoint();
}
return null;
}

Expand Down
6 changes: 6 additions & 0 deletions tests/integration-tests/pom.xml
Expand Up @@ -50,6 +50,12 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>artemis-quorum-ri</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.activemq.tests</groupId>
<artifactId>unit-tests</artifactId>
Expand Down
Expand Up @@ -53,6 +53,8 @@
import org.apache.activemq.artemis.core.server.cluster.ha.HAPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicaPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicatedPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicationBackupPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicationPrimaryPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.SharedStoreMasterPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.SharedStoreSlavePolicy;
import org.apache.activemq.artemis.core.server.files.FileMoveManager;
Expand Down Expand Up @@ -778,7 +780,7 @@ public void testFailBack() throws Exception {
((ReplicaPolicy) haPolicy).setMaxSavedReplicatedJournalsSize(1);
}

simpleFailover(haPolicy instanceof ReplicaPolicy, doFailBack);
simpleFailover(haPolicy instanceof ReplicaPolicy || haPolicy instanceof ReplicationBackupPolicy, doFailBack);
}

@Test(timeout = 120000)
Expand Down Expand Up @@ -808,7 +810,9 @@ public void testFailBackLiveRestartsBackupIsGone() throws Exception {
Thread.sleep(100);
Assert.assertFalse("backup is not running", backupServer.isStarted());

Assert.assertFalse("must NOT be a backup", liveServer.getServer().getHAPolicy() instanceof BackupPolicy);
final boolean isBackup = liveServer.getServer().getHAPolicy() instanceof BackupPolicy ||
liveServer.getServer().getHAPolicy() instanceof ReplicationBackupPolicy;
Assert.assertFalse("must NOT be a backup", isBackup);
adaptLiveConfigForReplicatedFailBack(liveServer);
beforeRestart(liveServer);
liveServer.start();
Expand All @@ -819,7 +823,8 @@ public void testFailBackLiveRestartsBackupIsGone() throws Exception {
ClientSession session2 = createSession(sf, false, false);
session2.start();
ClientConsumer consumer2 = session2.createConsumer(FailoverTestBase.ADDRESS);
boolean replication = liveServer.getServer().getHAPolicy() instanceof ReplicatedPolicy;
final boolean replication = liveServer.getServer().getHAPolicy() instanceof ReplicatedPolicy ||
liveServer.getServer().getHAPolicy() instanceof ReplicationPrimaryPolicy;
if (replication)
receiveMessages(consumer2, 0, NUM_MESSAGES, true);
assertNoMoreMessages(consumer2);
Expand All @@ -830,7 +835,7 @@ public void testFailBackLiveRestartsBackupIsGone() throws Exception {
public void testSimpleFailover() throws Exception {
HAPolicy haPolicy = backupServer.getServer().getHAPolicy();

simpleFailover(haPolicy instanceof ReplicaPolicy, false);
simpleFailover(haPolicy instanceof ReplicaPolicy || haPolicy instanceof ReplicationBackupPolicy, false);
}

@Test(timeout = 120000)
Expand Down
Expand Up @@ -19,6 +19,7 @@
import java.io.IOException;
import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
Expand All @@ -36,25 +37,33 @@
import org.apache.activemq.artemis.core.client.impl.ServerLocatorInternal;
import org.apache.activemq.artemis.core.config.ClusterConnectionConfiguration;
import org.apache.activemq.artemis.core.config.Configuration;
import org.apache.activemq.artemis.core.config.ha.DistributedPrimitiveManagerConfiguration;
import org.apache.activemq.artemis.core.config.ha.ReplicaPolicyConfiguration;
import org.apache.activemq.artemis.core.config.ha.SharedStoreMasterPolicyConfiguration;
import org.apache.activemq.artemis.core.config.ha.SharedStoreSlavePolicyConfiguration;
import org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnector;
import org.apache.activemq.artemis.core.remoting.impl.invm.InVMRegistry;
import org.apache.activemq.artemis.core.server.NodeManager;
import org.apache.activemq.artemis.core.server.cluster.ha.HAPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicatedPolicy;
import org.apache.activemq.artemis.core.server.cluster.ha.ReplicationPrimaryPolicy;
import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl;
import org.apache.activemq.artemis.core.server.impl.InVMNodeManager;
import org.apache.activemq.artemis.quorum.file.FileBasedPrimitiveManager;
import org.apache.activemq.artemis.tests.integration.cluster.util.SameProcessActiveMQServer;
import org.apache.activemq.artemis.tests.integration.cluster.util.TestableServer;
import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
import org.apache.activemq.artemis.tests.util.ReplicatedBackupUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;

public abstract class FailoverTestBase extends ActiveMQTestBase {
// Constants -----------------------------------------------------
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();

protected static final SimpleString ADDRESS = new SimpleString("FailoverTestAddress");

Expand Down Expand Up @@ -208,7 +217,34 @@ protected void createReplicatedConfigs() throws Exception {
liveServer = createTestableServer(liveConfig);
}

protected void createPluggableReplicatedConfigs() throws Exception {
final TransportConfiguration liveConnector = getConnectorTransportConfiguration(true);
final TransportConfiguration backupConnector = getConnectorTransportConfiguration(false);
final TransportConfiguration backupAcceptor = getAcceptorTransportConfiguration(false);

backupConfig = createDefaultInVMConfig();
liveConfig = createDefaultInVMConfig();

DistributedPrimitiveManagerConfiguration managerConfiguration =
new DistributedPrimitiveManagerConfiguration(FileBasedPrimitiveManager.class.getName(),
Collections.singletonMap("locks-folder", tmpFolder.newFolder("manager").toString()));

ReplicatedBackupUtils.configurePluggableQuorumReplicationPair(backupConfig, backupConnector, backupAcceptor, liveConfig, liveConnector, null, managerConfiguration, managerConfiguration);

backupConfig.setBindingsDirectory(getBindingsDir(0, true)).setJournalDirectory(getJournalDir(0, true)).setPagingDirectory(getPageDir(0, true)).setLargeMessagesDirectory(getLargeMessagesDir(0, true)).setSecurityEnabled(false);

setupHAPolicyConfiguration();
nodeManager = createReplicatedBackupNodeManager(backupConfig);

backupServer = createTestableServer(backupConfig);

liveConfig.clearAcceptorConfigurations().addAcceptorConfiguration(getAcceptorTransportConfiguration(true));

liveServer = createTestableServer(liveConfig);
}

protected void setupHAPolicyConfiguration() {
Assert.assertTrue(backupConfig.getHAPolicyConfiguration() instanceof ReplicaPolicyConfiguration);
((ReplicaPolicyConfiguration) backupConfig.getHAPolicyConfiguration()).setMaxSavedReplicatedJournalsSize(-1).setAllowFailBack(true);
((ReplicaPolicyConfiguration) backupConfig.getHAPolicyConfiguration()).setRestartBackup(false);
}
Expand All @@ -225,8 +261,13 @@ protected final void adaptLiveConfigForReplicatedFailBack(TestableServer server)
configuration.getConnectorConfigurations().put(backupConnector.getName(), backupConnector);
return;
}
ReplicatedPolicy haPolicy = (ReplicatedPolicy) server.getServer().getHAPolicy();
haPolicy.setCheckForLiveServer(true);
HAPolicy policy = server.getServer().getHAPolicy();
if (policy instanceof ReplicatedPolicy) {
((ReplicatedPolicy) policy).setCheckForLiveServer(true);
} else if (policy instanceof ReplicationPrimaryPolicy) {
Assert.assertTrue("Adapting won't work for the current configuration", ((ReplicationPrimaryPolicy) policy).isCheckForLiveServer());
}

}

@Override
Expand Down
@@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.integration.cluster.failover.quorum;

import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import org.apache.activemq.artemis.api.core.ActiveMQException;
import org.apache.activemq.artemis.api.core.Interceptor;
import org.apache.activemq.artemis.api.core.TransportConfiguration;
import org.apache.activemq.artemis.core.config.ha.ReplicationBackupPolicyConfiguration;
import org.apache.activemq.artemis.core.config.ha.ReplicationPrimaryPolicyConfiguration;
import org.apache.activemq.artemis.core.protocol.core.Packet;
import org.apache.activemq.artemis.core.protocol.core.impl.PacketImpl;
import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection;
import org.apache.activemq.artemis.tests.integration.cluster.failover.FailoverTestBase;
import org.apache.activemq.artemis.tests.integration.cluster.failover.FakeServiceComponent;
import org.apache.activemq.artemis.tests.util.TransportConfigurationUtils;
import org.apache.activemq.artemis.tests.util.Wait;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import static java.util.Arrays.asList;

@RunWith(Parameterized.class)
public class PluggableQuorumBackupAuthenticationTest extends FailoverTestBase {

private static CountDownLatch registrationStarted;

@Parameterized.Parameter
public boolean useNetty;

@Parameterized.Parameters(name = "useNetty={1}")
public static Iterable<Object[]> getParams() {
return asList(new Object[][]{{false}, {true}});
}

@Override
@Before
public void setUp() throws Exception {
startBackupServer = false;
registrationStarted = new CountDownLatch(1);
super.setUp();
}

@Test
public void testWrongPasswordSetting() throws Exception {
FakeServiceComponent fakeServiceComponent = new FakeServiceComponent("fake web server");
Wait.assertTrue(liveServer.getServer()::isActive);
waitForServerToStart(liveServer.getServer());
backupServer.start();
backupServer.getServer().addExternalComponent(fakeServiceComponent, true);
assertTrue(registrationStarted .await(5, TimeUnit.SECONDS));
/*
* can't intercept the message at the backup, so we intercept the registration message at the
* live.
*/
Wait.waitFor(() -> !backupServer.isStarted());
assertFalse("backup should have stopped", backupServer.isStarted());
Wait.assertFalse(fakeServiceComponent::isStarted);
backupServer.stop();
liveServer.stop();
}

@Override
protected void createConfigs() throws Exception {
createPluggableReplicatedConfigs();
backupConfig.setClusterPassword("crocodile");
liveConfig.setIncomingInterceptorClassNames(Arrays.asList(NotifyingInterceptor.class.getName()));
backupConfig.setSecurityEnabled(true);
liveConfig.setSecurityEnabled(true);
}

@Override
protected void setupHAPolicyConfiguration() {
((ReplicationPrimaryPolicyConfiguration) liveConfig.getHAPolicyConfiguration()).setCheckForLiveServer(true);
((ReplicationBackupPolicyConfiguration) backupConfig.getHAPolicyConfiguration()).setMaxSavedReplicatedJournalsSize(2).setAllowFailBack(true);
}

@Override
protected TransportConfiguration getAcceptorTransportConfiguration(final boolean live) {
return useNetty ? getNettyAcceptorTransportConfiguration(live) :
TransportConfigurationUtils.getInVMAcceptor(live);
}

@Override
protected TransportConfiguration getConnectorTransportConfiguration(final boolean live) {
return useNetty ? getNettyConnectorTransportConfiguration(live) :
TransportConfigurationUtils.getInVMConnector(live);
}

public static final class NotifyingInterceptor implements Interceptor {

@Override
public boolean intercept(Packet packet, RemotingConnection connection) throws ActiveMQException {
if (packet.getType() == PacketImpl.BACKUP_REGISTRATION) {
registrationStarted.countDown();
} else if (packet.getType() == PacketImpl.CLUSTER_CONNECT) {
registrationStarted.countDown();
}
return true;
}
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.activemq.artemis.tests.integration.cluster.failover.quorum;

import org.apache.activemq.artemis.core.config.ha.ReplicationBackupPolicyConfiguration;
import org.apache.activemq.artemis.core.config.ha.ReplicationPrimaryPolicyConfiguration;
import org.apache.activemq.artemis.tests.integration.cluster.failover.BackupSyncJournalTest;

public class PluggableQuorumBackupSyncJournalTest extends BackupSyncJournalTest {

@Override
protected void createConfigs() throws Exception {
createPluggableReplicatedConfigs();
}

@Override
protected void setupHAPolicyConfiguration() {
((ReplicationPrimaryPolicyConfiguration) liveConfig.getHAPolicyConfiguration())
.setCheckForLiveServer(true);
((ReplicationBackupPolicyConfiguration) backupConfig.getHAPolicyConfiguration())
.setMaxSavedReplicatedJournalsSize(2)
.setAllowFailBack(true);
}
}

0 comments on commit d2d53c6

Please sign in to comment.