From ca64c67acac41c34edf15ffa6ca8b5beac6f99ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 11 Feb 2022 14:26:54 +0100 Subject: [PATCH] [test] Replace PowerMockito#mockStatic usages with mockito-inline (#14098) * introduce NarClassLoaderBuilder since Mockito Inline cannot mock classloader classes * fix spyWithClassAndConstructorArgs(PulsarService.class, svcConfig) tests --- .../mledger/offload/OffloaderUtils.java | 16 +- .../mledger/offload/OffloadersCacheTest.java | 37 +- pom.xml | 12 + .../servlet/AdditionalServletUtils.java | 19 +- .../servlet/AdditionalServletUtilsTest.java | 71 +- .../intercept/BrokerInterceptorUtils.java | 18 +- .../broker/protocol/ProtocolHandlerUtils.java | 18 +- .../service/plugin/EntryFilterProvider.java | 19 +- .../intercept/BrokerInterceptorUtilsTest.java | 70 +- .../protocol/ProtocolHandlerUtilsTest.java | 102 +- ...sistentDispatcherFailoverConsumerTest.java | 5 +- .../broker/service/PersistentTopicTest.java | 5 +- .../pulsar/broker/service/ServerCnxTest.java | 5 +- ...ckyKeyDispatcherMultipleConsumersTest.java | 36 +- ...ckyKeyDispatcherMultipleConsumersTest.java | 92 +- .../PersistentSubscriptionTest.java | 5 +- ...dditionalServletWithPulsarServiceTest.java | 43 +- .../worker/PulsarFunctionLocalRunTest.java | 7 +- .../pulsar/admin/cli/CmdFunctionsTest.java | 24 - .../apache/pulsar/admin/cli/TestCmdSinks.java | 18 +- .../pulsar/admin/cli/TestCmdSources.java | 19 +- .../pulsar/admin/cli/TestCmdTopics.java | 13 - .../pulsar/common/nar/NarClassLoader.java | 18 +- .../common/nar/NarClassLoaderBuilder.java | 75 + .../thread/ThreadRuntimeFactoryTest.java | 90 +- .../functions/utils/FunctionCommon.java | 7 +- .../functions/utils/SinkConfigUtils.java | 16 +- .../functions/utils/SourceConfigUtils.java | 15 +- .../functioncache/FunctionCacheEntry.java | 10 +- .../utils/functions/FunctionUtils.java | 5 +- .../functions/utils/io/ConnectorUtils.java | 6 +- .../functions/utils/SinkConfigUtilsTest.java | 32 +- .../utils/SourceConfigUtilsTest.java | 33 +- .../worker/FunctionRuntimeManager.java | 6 +- .../worker/service/WorkerServiceLoader.java | 18 +- .../worker/ClusterServiceCoordinatorTest.java | 22 +- .../worker/FunctionRuntimeManagerTest.java | 1206 ++++++++--------- .../worker/rest/api/FunctionsImplTest.java | 10 - .../api/v2/FunctionApiV2ResourceTest.java | 208 ++- .../api/v3/FunctionApiV3ResourceTest.java | 193 ++- .../rest/api/v3/SinkApiV3ResourceTest.java | 459 +++---- .../rest/api/v3/SourceApiV3ResourceTest.java | 367 +++-- .../extensions/ProxyExtensionsUtils.java | 18 +- .../extensions/ProxyExtensionUtilsTest.java | 92 +- 44 files changed, 1645 insertions(+), 1915 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoaderBuilder.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java index f229603d6dcea..29689eb0b30e9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java @@ -24,13 +24,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; /** @@ -54,8 +54,11 @@ static Pair getOffloaderFactory(String n // need to load offloader NAR to the classloader that also loaded LedgerOffloaderFactory in case // LedgerOffloaderFactory is loaded by a classloader that is not the default classloader // as is the case for the pulsar presto plugin - NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - LedgerOffloaderFactory.class.getClassLoader(), narExtractionDirectory); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .parentClassLoader(LedgerOffloaderFactory.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); String configStr = ncl.getServiceDefinition(PULSAR_OFFLOADER_SERVICE_NAME); OffloaderDefinition conf = ObjectMapperFactory.getThreadLocalYaml() @@ -110,8 +113,11 @@ private static void rethrowIOException(Throwable cause) public static OffloaderDefinition getOffloaderDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .parentClassLoader(LedgerOffloaderFactory.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build()) { String configStr = ncl.getServiceDefinition(PULSAR_OFFLOADER_SERVICE_NAME); return ObjectMapperFactory.getThreadLocalYaml().readValue(configStr, OffloaderDefinition.class); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java index bf0c1c00f7cb3..5431a86481204 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java @@ -18,44 +18,33 @@ */ package org.apache.bookkeeper.mledger.offload; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; - import static org.mockito.ArgumentMatchers.eq; import static org.testng.Assert.assertSame; -@PrepareForTest({OffloaderUtils.class}) -@PowerMockIgnore({"org.apache.logging.log4j.*", "org.apache.pulsar.common.nar.*"}) public class OffloadersCacheTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadsOnlyOnce() throws Exception { Offloaders expectedOffloaders = new Offloaders(); - PowerMockito.mockStatic(OffloaderUtils.class); - PowerMockito.when(OffloaderUtils.searchForOffloaders(eq("./offloaders"), eq("/tmp"))) - .thenReturn(expectedOffloaders); + try (MockedStatic offloaderUtils = Mockito.mockStatic(OffloaderUtils.class)) { + offloaderUtils.when(() -> OffloaderUtils.searchForOffloaders(eq("./offloaders"), eq("/tmp"))) + .thenReturn(expectedOffloaders); - OffloadersCache cache = new OffloadersCache(); + OffloadersCache cache = new OffloadersCache(); - // Call a first time to load the offloader - Offloaders offloaders1 = cache.getOrLoadOffloaders("./offloaders", "/tmp"); + // Call a first time to load the offloader + Offloaders offloaders1 = cache.getOrLoadOffloaders("./offloaders", "/tmp"); - assertSame(offloaders1, expectedOffloaders, "The offloaders should be the mocked one."); + assertSame(offloaders1, expectedOffloaders, "The offloaders should be the mocked one."); - // Call a second time to get the stored offlaoder - Offloaders offloaders2 = cache.getOrLoadOffloaders("./offloaders", "/tmp"); + // Call a second time to get the stored offlaoder + Offloaders offloaders2 = cache.getOrLoadOffloaders("./offloaders", "/tmp"); - assertSame(offloaders2, expectedOffloaders, "The offloaders should be the mocked one."); + assertSame(offloaders2, expectedOffloaders, "The offloaders should be the mocked one."); + } } } diff --git a/pom.xml b/pom.xml index 472653027e91a..136d37fc2a149 100644 --- a/pom.xml +++ b/pom.xml @@ -316,6 +316,12 @@ flexible messaging model and an intuitive client API. ${mockito.version} + + org.mockito + mockito-inline + ${mockito.version} + + org.powermock powermock-api-mockito2 @@ -1298,6 +1304,12 @@ flexible messaging model and an intuitive client API. test + + org.mockito + mockito-inline + test + + org.powermock powermock-api-mockito2 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtils.java index e22dcb0815484..f7df7aaf7c39d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtils.java @@ -25,11 +25,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; /** @@ -50,9 +50,10 @@ public class AdditionalServletUtils { */ public AdditionalServletDefinition getAdditionalServletDefinition( String narPath, String narExtractionDirectory) throws IOException { - - try (NarClassLoader ncl = NarClassLoader.getFromArchive( - new File(narPath), Collections.emptySet(), narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build();) { return getAdditionalServletDefinition(ncl); } } @@ -118,10 +119,12 @@ public AdditionalServletDefinitions searchForServlets(String additionalServletDi public AdditionalServletWithClassLoader load( AdditionalServletMetadata metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - AdditionalServlet.class.getClassLoader(), narExtractionDirectory); + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(AdditionalServlet.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); AdditionalServletDefinition def = getAdditionalServletDefinition(ncl); if (StringUtils.isBlank(def.getAdditionalServletClass())) { diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java index bade954694870..c1929eb09208e 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java @@ -18,38 +18,24 @@ */ package org.apache.pulsar.broker.web.plugin.servlet; -import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.util.Set; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; -@PrepareForTest({ - NarClassLoader.class -}) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) public class AdditionalServletUtilsTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadEventListener() throws Exception { AdditionalServletDefinition def = new AdditionalServletDefinition(); @@ -69,19 +55,16 @@ public void testLoadEventListener() throws Exception { when(mockLoader.loadClass(eq(MockAdditionalServlet.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); - - AdditionalServletWithClassLoader returnedPhWithCL = AdditionalServletUtils.load(metadata, ""); - AdditionalServlet returnedPh = returnedPhWithCL.getServlet(); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); + AdditionalServletWithClassLoader returnedPhWithCL = AdditionalServletUtils.load(metadata, ""); + AdditionalServlet returnedPh = returnedPhWithCL.getServlet(); - assertSame(mockLoader, returnedPhWithCL.getClassLoader()); - assertTrue(returnedPh instanceof MockAdditionalServlet); + assertSame(mockLoader, returnedPhWithCL.getClassLoader()); + assertTrue(returnedPh instanceof MockAdditionalServlet); + } } @Test(expectedExceptions = IOException.class) @@ -102,15 +85,13 @@ public void testLoadEventListenerWithBlankListerClass() throws Exception { when(mockLoader.loadClass(eq(MockAdditionalServlet.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - AdditionalServletUtils.load(metadata, ""); + AdditionalServletUtils.load(metadata, ""); + } } @Test(expectedExceptions = IOException.class) @@ -132,14 +113,12 @@ public void testLoadEventListenerWithWrongListerClass() throws Exception { when(mockLoader.loadClass(eq(Runnable.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - AdditionalServletUtils.load(metadata, ""); + AdditionalServletUtils.load(metadata, ""); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtils.java index 9f7f502205dc6..96421e8b508e2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtils.java @@ -25,11 +25,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; /** @@ -50,8 +50,10 @@ public class BrokerInterceptorUtils { */ public BrokerInterceptorDefinition getBrokerInterceptorDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build()) { return getBrokerInterceptorDefinition(ncl); } } @@ -117,10 +119,12 @@ public BrokerInterceptorDefinitions searchForInterceptors(String interceptorsDir */ BrokerInterceptorWithClassLoader load(BrokerInterceptorMetadata metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - BrokerInterceptor.class.getClassLoader(), narExtractionDirectory); + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(BrokerInterceptorUtils.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); BrokerInterceptorDefinition def = getBrokerInterceptorDefinition(ncl); if (StringUtils.isBlank(def.getInterceptorClass())) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtils.java index 4a05bd67fe6dc..c65bf3713aebe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtils.java @@ -25,11 +25,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; /** @@ -50,8 +50,10 @@ class ProtocolHandlerUtils { */ public static ProtocolHandlerDefinition getProtocolHandlerDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build()) { return getProtocolHandlerDefinition(ncl); } } @@ -117,10 +119,12 @@ public static ProtocolHandlerDefinitions searchForHandlers(String handlersDirect */ static ProtocolHandlerWithClassLoader load(ProtocolHandlerMetadata metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - ProtocolHandler.class.getClassLoader(), narExtractionDirectory); + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(ProtocolHandler.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); ProtocolHandlerDefinition phDef = getProtocolHandlerDefinition(ncl); if (StringUtils.isBlank(phDef.getHandlerClass())) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java index 9e19a5784d3fc..d92796521add8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java @@ -26,11 +26,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; @Slf4j @@ -104,8 +104,10 @@ private static EntryFilterDefinitions searchForEntryFilters(String entryFiltersD private static EntryFilterDefinition getEntryFilterDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build()) { return getEntryFilterDefinition(ncl); } } @@ -121,11 +123,12 @@ private static EntryFilterDefinition getEntryFilterDefinition(NarClassLoader ncl private static EntryFilterWithClassLoader load(EntryFilterMetaData metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - EntryFilter.class.getClassLoader(), narExtractionDirectory); - + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(EntryFilter.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); EntryFilterDefinition def = getEntryFilterDefinition(ncl); if (StringUtils.isBlank(def.getEntryFilterClass())) { throw new IOException("Entry filters `" + def.getName() + "` does NOT provide a entry" diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java index 4b7c1e5ddc4de..c618501f23fa2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java @@ -19,39 +19,25 @@ package org.apache.pulsar.broker.intercept; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; -import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.util.Set; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; -@PrepareForTest({ - BrokerInterceptorUtils.class, NarClassLoader.class -}) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) @Test(groups = "broker") public class BrokerInterceptorUtilsTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadBrokerEventListener() throws Exception { BrokerInterceptorDefinition def = new BrokerInterceptorDefinition(); @@ -71,19 +57,17 @@ public void testLoadBrokerEventListener() throws Exception { when(mockLoader.loadClass(eq(MockBrokerInterceptor.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - BrokerInterceptorWithClassLoader returnedPhWithCL = BrokerInterceptorUtils.load(metadata, ""); - BrokerInterceptor returnedPh = returnedPhWithCL.getInterceptor(); + BrokerInterceptorWithClassLoader returnedPhWithCL = BrokerInterceptorUtils.load(metadata, ""); + BrokerInterceptor returnedPh = returnedPhWithCL.getInterceptor(); - assertSame(mockLoader, returnedPhWithCL.getClassLoader()); - assertTrue(returnedPh instanceof MockBrokerInterceptor); + assertSame(mockLoader, returnedPhWithCL.getClassLoader()); + assertTrue(returnedPh instanceof MockBrokerInterceptor); + } } @Test(expectedExceptions = IOException.class) @@ -104,15 +88,13 @@ public void testLoadBrokerEventListenerWithBlankListerClass() throws Exception { when(mockLoader.loadClass(eq(MockBrokerInterceptor.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - BrokerInterceptorUtils.load(metadata, ""); + BrokerInterceptorUtils.load(metadata, ""); + } } @Test(expectedExceptions = IOException.class) @@ -134,14 +116,12 @@ public void testLoadBrokerEventListenerWithWrongListerClass() throws Exception { when(mockLoader.loadClass(eq(Runnable.class.getName()))) .thenReturn(listenerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - BrokerInterceptorUtils.load(metadata, ""); + BrokerInterceptorUtils.load(metadata, ""); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtilsTest.java index 60a802f230168..e1fed531c7761 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtilsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/ProtocolHandlerUtilsTest.java @@ -19,40 +19,26 @@ package org.apache.pulsar.broker.protocol; import static org.apache.pulsar.broker.protocol.ProtocolHandlerUtils.PULSAR_PROTOCOL_HANDLER_DEFINITION_FILE; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; import static org.testng.AssertJUnit.fail; -import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.util.Set; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; -@PrepareForTest({ - ProtocolHandlerUtils.class, NarClassLoader.class -}) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) @Test(groups = "broker") public class ProtocolHandlerUtilsTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadProtocolHandler() throws Exception { ProtocolHandlerDefinition def = new ProtocolHandlerDefinition(); @@ -72,19 +58,17 @@ public void testLoadProtocolHandler() throws Exception { when(mockLoader.loadClass(eq(MockProtocolHandler.class.getName()))) .thenReturn(handlerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - ProtocolHandlerWithClassLoader returnedPhWithCL = ProtocolHandlerUtils.load(metadata, ""); - ProtocolHandler returnedPh = returnedPhWithCL.getHandler(); + ProtocolHandlerWithClassLoader returnedPhWithCL = ProtocolHandlerUtils.load(metadata, ""); + ProtocolHandler returnedPh = returnedPhWithCL.getHandler(); - assertSame(mockLoader, returnedPhWithCL.getClassLoader()); - assertTrue(returnedPh instanceof MockProtocolHandler); + assertSame(mockLoader, returnedPhWithCL.getClassLoader()); + assertTrue(returnedPh instanceof MockProtocolHandler); + } } @Test @@ -100,24 +84,22 @@ public void testLoadProtocolHandlerBlankHandlerClass() throws Exception { NarClassLoader mockLoader = mock(NarClassLoader.class); when(mockLoader.getServiceDefinition(eq(PULSAR_PROTOCOL_HANDLER_DEFINITION_FILE))) - .thenReturn(ObjectMapperFactory.getThreadLocalYaml().writeValueAsString(def)); + .thenReturn(ObjectMapperFactory.getThreadLocalYaml().writeValueAsString(def)); Class handlerClass = MockProtocolHandler.class; when(mockLoader.loadClass(eq(MockProtocolHandler.class.getName()))) - .thenReturn(handlerClass); - - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); - - try { - ProtocolHandlerUtils.load(metadata, ""); - fail("Should not reach here"); - } catch (IOException ioe) { - // expected + .thenReturn(handlerClass); + + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); + + try { + ProtocolHandlerUtils.load(metadata, ""); + fail("Should not reach here"); + } catch (IOException ioe) { + // expected + } } } @@ -135,24 +117,22 @@ public void testLoadProtocolHandlerWrongHandlerClass() throws Exception { NarClassLoader mockLoader = mock(NarClassLoader.class); when(mockLoader.getServiceDefinition(eq(PULSAR_PROTOCOL_HANDLER_DEFINITION_FILE))) - .thenReturn(ObjectMapperFactory.getThreadLocalYaml().writeValueAsString(def)); + .thenReturn(ObjectMapperFactory.getThreadLocalYaml().writeValueAsString(def)); Class handlerClass = Runnable.class; when(mockLoader.loadClass(eq(Runnable.class.getName()))) - .thenReturn(handlerClass); - - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); - - try { - ProtocolHandlerUtils.load(metadata, ""); - fail("Should not reach here"); - } catch (IOException ioe) { - // expected + .thenReturn(handlerClass); + + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); + + try { + ProtocolHandlerUtils.load(metadata, ""); + fail("Should not reach here"); + } catch (IOException ioe) { + // expected + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index d76505a69808d..5700b1c42f1ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -115,6 +115,7 @@ public void setup() throws Exception { executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-dispatcher-failover-test").build(); ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); + svcConfig.setClusterName("pulsar-cluster"); pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); @@ -200,7 +201,9 @@ public void shutdown() throws Exception { } executor.shutdown(); - eventLoopGroup.shutdownGracefully().get(); + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().get(); + } store.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index fb30344a1cb5a..68e0f4f77c8a1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -176,6 +176,7 @@ public void setup() throws Exception { svcConfig.setAdvertisedAddress("localhost"); svcConfig.setBrokerShutdownTimeoutMs(0L); svcConfig.setMaxUnackedMessagesPerConsumer(50000); + svcConfig.setClusterName("pulsar-cluster"); pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(svcConfig).when(pulsar).getConfiguration(); doReturn(mock(Compactor.class)).when(pulsar).getCompactor(); @@ -244,7 +245,9 @@ public void teardown() throws Exception { } executor.shutdownNow(); - eventLoopGroup.shutdownGracefully().get(); + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().get(); + } } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index a27bc8bf2bad9..f008ec4f57273 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -154,6 +154,7 @@ public void setup() throws Exception { executor = OrderedExecutor.newBuilder().numThreads(1).build(); svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); + svcConfig.setClusterName("pulsar-cluster"); pulsar = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); doReturn(new DefaultSchemaRegistryService()).when(pulsar).getSchemaRegistryService(); @@ -210,7 +211,9 @@ public void teardown() throws Exception { pulsar.close(); brokerService.close(); executor.shutdownNow(); - eventLoopGroup.shutdownGracefully().get(); + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().get(); + } store.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java index 1492fe0030148..f319b7ce4a7e1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -53,16 +54,10 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.protocol.Commands; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; +import org.mockito.MockedStatic; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; -@PrepareForTest({ DispatchRateLimiter.class }) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) public class NonPersistentStickyKeyDispatcherMultipleConsumersTest { private PulsarService pulsarMock; @@ -75,11 +70,6 @@ public class NonPersistentStickyKeyDispatcherMultipleConsumersTest { final String topicName = "non-persistent://public/default/testTopic"; - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @BeforeMethod public void setup() throws Exception { configMock = mock(ServiceConfiguration.class); @@ -104,17 +94,17 @@ public void setup() throws Exception { subscriptionMock = mock(NonPersistentSubscription.class); - PowerMockito.mockStatic(DispatchRateLimiter.class); - PowerMockito.when(DispatchRateLimiter.isDispatchRateNeeded( - any(BrokerService.class), - any(Optional.class), - anyString(), - any(DispatchRateLimiter.Type.class)) - ).thenReturn(false); - - nonpersistentDispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers( - topicMock, subscriptionMock, - new HashRangeAutoSplitStickyKeyConsumerSelector()); + try (MockedStatic rateLimiterMockedStatic = mockStatic(DispatchRateLimiter.class);) { + rateLimiterMockedStatic.when(() -> DispatchRateLimiter.isDispatchRateNeeded( + any(BrokerService.class), + any(Optional.class), + anyString(), + any(DispatchRateLimiter.Type.class))) + .thenReturn(false); + nonpersistentDispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers( + topicMock, subscriptionMock, + new HashRangeAutoSplitStickyKeyConsumerSelector()); + } } @Test(timeOut = 10000) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index db51af4cb7c43..99a66f44ac4c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -32,6 +32,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -71,17 +72,11 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; import org.mockito.ArgumentCaptor; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.mockito.MockedStatic; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; -@PrepareForTest({ DispatchRateLimiter.class }) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) @Test(groups = "broker") public class PersistentStickyKeyDispatcherMultipleConsumersTest { @@ -99,11 +94,6 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest { final String topicName = "persistent://public/default/testTopic"; final String subscriptionName = "testSubscription"; - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @BeforeMethod public void setup() throws Exception { configMock = mock(ServiceConfiguration.class); @@ -153,18 +143,17 @@ public void setup() throws Exception { ); subscriptionMock = mock(PersistentSubscription.class); - - PowerMockito.mockStatic(DispatchRateLimiter.class); - PowerMockito.when(DispatchRateLimiter.isDispatchRateNeeded( - any(BrokerService.class), - any(Optional.class), - anyString(), - any(DispatchRateLimiter.Type.class)) - ).thenReturn(false); - - persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( - topicMock, cursorMock, subscriptionMock, configMock, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)); + try (MockedStatic rateLimiterMockedStatic = mockStatic(DispatchRateLimiter.class);) { + rateLimiterMockedStatic.when(() -> DispatchRateLimiter.isDispatchRateNeeded( + any(BrokerService.class), + any(Optional.class), + anyString(), + any(DispatchRateLimiter.Type.class))) + .thenReturn(false); + persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)); + } } @Test @@ -208,31 +197,40 @@ public void testSendMarkerMessage() { @Test(timeOut = 10000) public void testSendMessage() { - KeySharedMeta keySharedMeta = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY); - PersistentStickyKeyDispatcherMultipleConsumers persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( - topicMock, cursorMock, subscriptionMock, configMock, keySharedMeta); - try { - keySharedMeta.addHashRange() - .setStart(0) - .setEnd(9); - - Consumer consumerMock = mock(Consumer.class); - doReturn(keySharedMeta).when(consumerMock).getKeySharedMeta(); - persistentDispatcher.addConsumer(consumerMock); - persistentDispatcher.consumerFlow(consumerMock, 1000); - } catch (Exception e) { - fail("Failed to add mock consumer", e); - } + try (MockedStatic rateLimiterMockedStatic = mockStatic(DispatchRateLimiter.class);) { + rateLimiterMockedStatic.when(() -> DispatchRateLimiter.isDispatchRateNeeded( + any(BrokerService.class), + any(Optional.class), + anyString(), + any(DispatchRateLimiter.Type.class))) + .thenReturn(false); + KeySharedMeta keySharedMeta = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY); + DispatchRateLimiter.isDispatchRateNeeded(brokerMock, Optional.empty(), "hello", DispatchRateLimiter.Type.SUBSCRIPTION); + PersistentStickyKeyDispatcherMultipleConsumers persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, keySharedMeta); + try { + keySharedMeta.addHashRange() + .setStart(0) + .setEnd(9); + + Consumer consumerMock = mock(Consumer.class); + doReturn(keySharedMeta).when(consumerMock).getKeySharedMeta(); + persistentDispatcher.addConsumer(consumerMock); + persistentDispatcher.consumerFlow(consumerMock, 1000); + } catch (Exception e) { + fail("Failed to add mock consumer", e); + } - List entries = new ArrayList<>(); - entries.add(EntryImpl.create(1, 1, createMessage("message1", 1))); - entries.add(EntryImpl.create(1, 2, createMessage("message2", 2))); + List entries = new ArrayList<>(); + entries.add(EntryImpl.create(1, 1, createMessage("message1", 1))); + entries.add(EntryImpl.create(1, 2, createMessage("message2", 2))); - try { - //Should success,see issue #8960 - persistentDispatcher.readEntriesComplete(entries, PersistentStickyKeyDispatcherMultipleConsumers.ReadType.Normal); - } catch (Exception e) { - fail("Failed to readEntriesComplete.", e); + try { + //Should success,see issue #8960 + persistentDispatcher.readEntriesComplete(entries, PersistentStickyKeyDispatcherMultipleConsumers.ReadType.Normal); + } catch (Exception e) { + fail("Failed to readEntriesComplete.", e); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index b9304cb5fb8ea..c0ca352c15300 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -113,6 +113,7 @@ public void setup() throws Exception { ServiceConfiguration svcConfig = spy(ServiceConfiguration.class); svcConfig.setBrokerShutdownTimeoutMs(0L); svcConfig.setTransactionCoordinatorEnabled(true); + svcConfig.setClusterName("pulsar-cluster"); pulsarMock = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig); PulsarResources pulsarResources = mock(PulsarResources.class); doReturn(pulsarResources).when(pulsarMock).getPulsarResources(); @@ -216,7 +217,9 @@ public void teardown() throws Exception { store.close(); executor.shutdownNow(); - eventLoopGroup.shutdownGracefully().get(); + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().get(); + } } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithPulsarServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithPulsarServiceTest.java index 479c9435370e7..fcc806fc35a48 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithPulsarServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithPulsarServiceTest.java @@ -18,37 +18,22 @@ */ package org.apache.pulsar.broker.web.plugin.servlet; - -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; -import java.io.File; import java.nio.file.Paths; -import java.util.Set; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; -@PrepareForTest({ - NarClassLoader.class -}) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) public class AdditionalServletWithPulsarServiceTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadAdditionalServlet() throws Exception { AdditionalServletDefinition def = new AdditionalServletDefinition(); @@ -68,18 +53,16 @@ public void testLoadAdditionalServlet() throws Exception { when(mockLoader.loadClass(eq(MockAdditionalServletWithClassLoader.class.getName()))) .thenReturn(additionalServletClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - AdditionalServletWithClassLoader returnedASWithCL = AdditionalServletUtils.load(metadata, ""); - AdditionalServlet returnedPh = returnedASWithCL.getServlet(); + AdditionalServletWithClassLoader returnedASWithCL = AdditionalServletUtils.load(metadata, ""); + AdditionalServlet returnedPh = returnedASWithCL.getServlet(); - assertSame(mockLoader, returnedASWithCL.getClassLoader()); - assertTrue(returnedPh instanceof MockAdditionalServletWithClassLoader); + assertSame(mockLoader, returnedASWithCL.getClassLoader()); + assertTrue(returnedPh instanceof MockAdditionalServletWithClassLoader); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index 14db9a4b577a0..5592e6975cddf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -74,6 +74,7 @@ import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ConsumerStats; import org.apache.pulsar.common.policies.data.PublisherStats; @@ -1120,7 +1121,11 @@ public void testExitOnError() throws Throwable{ private void runWithNarClassLoader(Assert.ThrowingRunnable throwingRunnable) throws Throwable { ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader(); - try (NarClassLoader classLoader = NarClassLoader.getFromArchive(getPulsarIODataGeneratorNar(), Collections.emptySet(), originalClassLoader, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR)) { + try (NarClassLoader classLoader = NarClassLoaderBuilder.builder() + .narFile(getPulsarIODataGeneratorNar()) + .parentClassLoader(originalClassLoader) + .extractionDirectory(NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR) + .build()) { try { Thread.currentThread().setContextClassLoader(classLoader); throwingRunnable.run(); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index 3e32961d8040f..5ea9b69a43bcd 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -26,20 +26,17 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.util.Arrays; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.clients.StorageClientBuilder; import org.apache.pulsar.admin.cli.CmdFunctions.CreateFunction; import org.apache.pulsar.admin.cli.CmdFunctions.DeleteFunction; import org.apache.pulsar.admin.cli.CmdFunctions.GetFunction; @@ -56,28 +53,15 @@ import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.utils.IdentityFunction; -import org.apache.pulsar.common.util.Reflections; -import org.apache.pulsar.functions.utils.FunctionCommon; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link CmdFunctions}. */ @Slf4j -@PrepareForTest({ CmdFunctions.class, Reflections.class, StorageClientBuilder.class, FunctionCommon.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*" }) public class CmdFunctionsTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final String TEST_NAME = "test_name"; private static final String JAR_NAME = CmdFunctionsTest.class.getClassLoader().getResource("dummyexamples.jar").getFile(); private static final String GO_EXEC_FILE_NAME = "test-go-function-with-url"; @@ -123,14 +107,6 @@ public void setup() throws Exception { this.cmdSinks = new CmdSinks(() -> admin); this.cmdSources = new CmdSources(() -> admin); - // mock reflections - mockStatic(Reflections.class); - when(Reflections.classExistsInJar(any(File.class), anyString())).thenReturn(true); - when(Reflections.classExists(anyString())).thenReturn(true); - when(Reflections.classInJarImplementsIface(any(File.class), anyString(), eq(Function.class))) - .thenReturn(true); - when(Reflections.classImplementsIface(anyString(), any())).thenReturn(true); - when(Reflections.createInstance(eq(DummyFunction.class.getName()), any(File.class))).thenReturn(new DummyFunction()); } // @Test diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java index 27402bddbf0e8..1b67b18a21024 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java @@ -25,8 +25,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mockStatic; - import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; @@ -48,26 +46,15 @@ import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.util.ClassLoaderUtils; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.mockito.Mockito; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; @Slf4j -@PrepareForTest({CmdFunctions.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.io.core.*", "org.apache.pulsar.functions.api.*" }) public class TestCmdSinks { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final String TENANT = "test-tenant"; private static final String NAMESPACE = "test-namespace"; private static final String NAME = "test"; @@ -121,8 +108,7 @@ public void setup() throws Exception { localSinkRunner = spy(cmdSinks.getLocalSinkRunner()); deleteSink = spy(cmdSinks.getDeleteSink()); - mockStatic(CmdFunctions.class); - PowerMockito.doNothing().when(localSinkRunner).runCmd(); + Mockito.doNothing().when(localSinkRunner).runCmd(); URL file = Thread.currentThread().getContextClassLoader().getResource(JAR_FILE_NAME); if (file == null) { throw new RuntimeException("Failed to file required test archive: " + JAR_FILE_NAME); diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java index ff75d26534153..fe9075540638c 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java @@ -25,7 +25,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -45,26 +44,15 @@ import org.apache.pulsar.common.io.BatchSourceConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.util.ClassLoaderUtils; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.mockito.Mockito; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; -@PrepareForTest({CmdFunctions.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.io.core.*" }) public class TestCmdSources { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - - private static final String TENANT = "test-tenant"; + private static final String TENANT = "test-tenant"; private static final String NAMESPACE = "test-namespace"; private static final String NAME = "test"; private static final String TOPIC_NAME = "src_topic_1"; @@ -105,8 +93,7 @@ public void setup() throws Exception { localSourceRunner = spy(CmdSources.getLocalSourceRunner()); deleteSource = spy(CmdSources.getDeleteSource()); - mockStatic(CmdFunctions.class); - PowerMockito.doNothing().when(localSourceRunner).runCmd(); + Mockito.doNothing().when(localSourceRunner).runCmd(); JAR_FILE_PATH = Thread.currentThread().getContextClassLoader().getResource(JAR_FILE_NAME).getFile(); jarClassLoader = ClassLoaderUtils.loadJar(new File(JAR_FILE_PATH)); oldContextClassLoader = Thread.currentThread().getContextClassLoader(); diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java index e723c77699f3b..dfef399587d00 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java @@ -28,7 +28,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import com.google.common.collect.Lists; import java.io.ByteArrayOutputStream; @@ -45,23 +44,13 @@ import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo; import org.mockito.Mockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; -@PrepareForTest({CmdFunctions.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.io.core.*" }) public class TestCmdTopics { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } private static final String PERSISTENT_TOPIC_URL = "persistent://"; private static final String PARTITIONED_TOPIC_NAME = "my-topic"; private static final String URL_SLASH = "/"; @@ -81,8 +70,6 @@ public void setup() throws Exception { when(pulsarAdmin.lookups()).thenReturn(mockLookup); cmdTopics = spy(new CmdTopics(() -> pulsarAdmin)); partitionedLookup = spy(cmdTopics.getPartitionedLookup()); - - mockStatic(CmdFunctions.class); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java index 8c82b2fa9f0be..9f5a0ee6bfaa1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java @@ -141,17 +141,7 @@ public class NarClassLoader extends URLClassLoader { public static final String DEFAULT_NAR_EXTRACTION_DIR = System.getProperty("java.io.tmpdir"); - public static NarClassLoader getFromArchive(File narPath, Set additionalJars, - String narExtractionDirectory) throws IOException { - return NarClassLoader.getFromArchive(narPath, additionalJars, NarClassLoader.class.getClassLoader(), - narExtractionDirectory); - } - - public static NarClassLoader getFromArchive(File narPath, Set additionalJars) throws IOException { - return NarClassLoader.getFromArchive(narPath, additionalJars, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); - } - - public static NarClassLoader getFromArchive(File narPath, Set additionalJars, ClassLoader parent, + static NarClassLoader getFromArchive(File narPath, Set additionalJars, ClassLoader parent, String narExtractionDirectory) throws IOException { File unpacked = NarUnpacker.unpackNar(narPath, getNarExtractionDirectory(narExtractionDirectory)); @@ -190,8 +180,10 @@ private NarClassLoader(final File narWorkingDirectory, Set additionalJar // process the classpath updateClasspath(narWorkingDirectory); - for (String jar : additionalJars) { - addURL(Paths.get(jar).toUri().toURL()); + if (additionalJars != null) { + for (String jar : additionalJars) { + addURL(Paths.get(jar).toUri().toURL()); + } } if (log.isDebugEnabled()) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoaderBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoaderBuilder.java new file mode 100644 index 0000000000000..a9ffd754e456e --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoaderBuilder.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * This class was adapted from NiFi NAR Utils + * https://github.com/apache/nifi/tree/master/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils + */ +package org.apache.pulsar.common.nar; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +/** + * NarClassLoader builder class. + */ +public class NarClassLoaderBuilder { + + private File narFile; + private Set additionalJars; + private ClassLoader parentClassLoader; + private String extractionDirectory; + + public static NarClassLoaderBuilder builder() { + return new NarClassLoaderBuilder(); + } + + public NarClassLoaderBuilder narFile(File narFile) { + this.narFile = narFile; + return this; + } + + public NarClassLoaderBuilder additionalJars(Set additionalJars) { + this.additionalJars = additionalJars; + return this; + } + + public NarClassLoaderBuilder parentClassLoader(ClassLoader parentClassLoader) { + this.parentClassLoader = parentClassLoader; + return this; + } + public NarClassLoaderBuilder extractionDirectory(String extractionDirectory) { + this.extractionDirectory = extractionDirectory; + return this; + } + + public NarClassLoader build() throws IOException { + if (parentClassLoader == null) { + parentClassLoader = NarClassLoader.class.getClassLoader(); + } + if (extractionDirectory == null) { + extractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; + } + Objects.requireNonNull(narFile); + return NarClassLoader.getFromArchive(narFile, additionalJars, parentClassLoader, extractionDirectory); + } + +} diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java index 8500fb9ecd718..c86e1d5f2d6ff 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java @@ -26,31 +26,23 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.worker.ConnectorsManager; import org.apache.pulsar.functions.worker.WorkerConfig; +import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.powermock.reflect.Whitebox; import org.testng.annotations.Test; - import java.util.Map; import java.util.Optional; -@PrepareForTest({InstanceUtils.class, PulsarClient.class, PlatformDependent.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*"}) +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mockStatic; + @Slf4j public class ThreadRuntimeFactoryTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testMemoryLimitPercent() throws Exception { @@ -112,42 +104,40 @@ public void testMemoryLimitBothSet() throws Exception { private ClientBuilder testMemoryLimit(Long absolute, Double percent) throws Exception { - PowerMockito.mockStatic(PulsarClient.class); - PowerMockito.mockStatic(PlatformDependent.class); - - PowerMockito.when(PlatformDependent.maxDirectMemory()).thenReturn(1024L); - - ClientBuilder clientBuilder = Mockito.mock(ClientBuilder.class); - PowerMockito.when(PulsarClient.builder()).thenReturn(clientBuilder); - PowerMockito.when(PulsarClient.builder().serviceUrl(Mockito.anyString())).thenReturn(clientBuilder); - - - ThreadRuntimeFactoryConfig threadRuntimeFactoryConfig = new ThreadRuntimeFactoryConfig(); - threadRuntimeFactoryConfig.setThreadGroupName("foo"); - ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit(); - if (percent != null) { - memoryLimit.setPercentOfMaxDirectMemory(percent); - } - - if (absolute != null) { - memoryLimit.setAbsoluteValue(absolute); + try (MockedStatic mockedPulsarClient = mockStatic(PulsarClient.class);) { + Whitebox.setInternalState(PlatformDependent.class, "DIRECT_MEMORY_LIMIT", 1024L); + + ClientBuilder clientBuilder = Mockito.mock(ClientBuilder.class); + mockedPulsarClient.when(() -> PulsarClient.builder()).thenAnswer(i -> clientBuilder); + doReturn(clientBuilder).when(clientBuilder).serviceUrl(anyString()); + + ThreadRuntimeFactoryConfig threadRuntimeFactoryConfig = new ThreadRuntimeFactoryConfig(); + threadRuntimeFactoryConfig.setThreadGroupName("foo"); + ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit(); + if (percent != null) { + memoryLimit.setPercentOfMaxDirectMemory(percent); + } + + if (absolute != null) { + memoryLimit.setAbsoluteValue(absolute); + } + threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(memoryLimit); + + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName()); + workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(threadRuntimeFactoryConfig, Map.class)); + workerConfig.setPulsarServiceUrl("pulsar://broker.pulsar:6650"); + + ThreadRuntimeFactory threadRuntimeFactory = new ThreadRuntimeFactory(); + + threadRuntimeFactory.initialize( + workerConfig, + Mockito.mock(AuthenticationConfig.class), + Mockito.mock(SecretsProviderConfigurator.class), + Mockito.mock(ConnectorsManager.class), + Optional.empty(), Optional.empty()); + + return clientBuilder; } - threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(memoryLimit); - - WorkerConfig workerConfig = new WorkerConfig(); - workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName()); - workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(threadRuntimeFactoryConfig, Map.class)); - workerConfig.setPulsarServiceUrl("pulsar://broker.pulsar:6650"); - - ThreadRuntimeFactory threadRuntimeFactory = new ThreadRuntimeFactory(); - - threadRuntimeFactory.initialize( - workerConfig, - Mockito.mock(AuthenticationConfig.class), - Mockito.mock(SecretsProviderConfigurator.class), - Mockito.mock(ConnectorsManager.class), - Optional.empty(), Optional.empty()); - - return clientBuilder; } } \ No newline at end of file diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index f72814df9905c..a960dadc4b134 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -52,6 +52,7 @@ import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.WindowFunction; @@ -260,8 +261,10 @@ public static NarClassLoader extractNarClassLoader(File packageFile, String narExtractionDirectory) { if (packageFile != null) { try { - return NarClassLoader.getFromArchive(packageFile, - Collections.emptySet(), narExtractionDirectory); + return NarClassLoaderBuilder.builder() + .narFile(packageFile) + .extractionDirectory(narExtractionDirectory) + .build(); } catch (IOException e) { throw new IllegalArgumentException(e.getMessage()); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 4978b317bc1f7..1d567614e6aec 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -625,16 +625,22 @@ public static void validateSinkConfig(SinkConfig sinkConfig, NarClassLoader narC ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); if (defn.getSinkConfigClass() != null) { Class configClass = Class.forName(defn.getSinkConfigClass(), true, narClassLoader); - Object configObject = - ObjectMapperFactory.getThreadLocal().convertValue(sinkConfig.getConfigs(), configClass); - if (configObject != null) { - ConfigValidation.validateConfig(configObject); - } + validateSinkConfig(sinkConfig, configClass); } } catch (IOException e) { throw new IllegalArgumentException("Error validating sink config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find sink config class", e); + } + } + + public static void validateSinkConfig(SinkConfig sinkConfig, Class configClass) { + try { + Object configObject = + ObjectMapperFactory.getThreadLocal().convertValue(sinkConfig.getConfigs(), configClass); + if (configObject != null) { + ConfigValidation.validateConfig(configObject); + } } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not validate sink config: " + e.getMessage()); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java index 6215fe36f4789..c49ac22e93cf4 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java @@ -512,15 +512,22 @@ public static void validateSourceConfig(SourceConfig sourceConfig, NarClassLoade ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); if (defn.getSourceConfigClass() != null) { Class configClass = Class.forName(defn.getSourceConfigClass(), true, narClassLoader); - Object configObject = ObjectMapperFactory.getThreadLocal().convertValue(sourceConfig.getConfigs(), configClass); - if (configObject != null) { - ConfigValidation.validateConfig(configObject); - } + validateSourceConfig(sourceConfig, configClass); } } catch (IOException e) { throw new IllegalArgumentException("Error validating source config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find source config class"); + } + + } + + public static void validateSourceConfig(SourceConfig sourceConfig, Class configClass) { + try { + Object configObject = ObjectMapperFactory.getThreadLocal().convertValue(sourceConfig.getConfigs(), configClass); + if (configObject != null) { + ConfigValidation.validateConfig(configObject); + } } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not validate source config: " + e.getMessage()); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functioncache/FunctionCacheEntry.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functioncache/FunctionCacheEntry.java index e9fb409567aa7..0a67515cf1917 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functioncache/FunctionCacheEntry.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functioncache/FunctionCacheEntry.java @@ -33,7 +33,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; /** * A cache entry in the function cache. Tracks which workers still reference @@ -70,8 +70,12 @@ public class FunctionCacheEntry implements AutoCloseable { FunctionCacheEntry(String narArchive, String initialInstanceId, ClassLoader rootClassLoader, String narExtractionDirectory) throws IOException { - this.classLoader = NarClassLoader.getFromArchive(new File(narArchive), Collections.emptySet(), - rootClassLoader, narExtractionDirectory); + + this.classLoader = NarClassLoaderBuilder.builder() + .narFile(new File(narArchive)) + .extractionDirectory(narExtractionDirectory) + .parentClassLoader(rootClassLoader) + .build(); this.classpaths = Collections.emptySet(); this.jarFiles = Collections.singleton(narArchive); this.executionHolders = new HashSet<>(Collections.singleton(initialInstanceId)); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index 2493e1b2d95a5..fb4e20b7be543 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -32,6 +32,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.utils.Exceptions; import org.apache.pulsar.functions.api.Function; @@ -72,7 +73,9 @@ public static String getFunctionClass(ClassLoader classLoader) throws IOExceptio } public static FunctionDefinition getFunctionDefinition(String narPath) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet())) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .build();) { String configStr = ncl.getServiceDefinition(PULSAR_IO_SERVICE_NAME); return ObjectMapperFactory.getThreadLocalYaml().readValue(configStr, FunctionDefinition.class); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index 39cfc25a45aba..4b444f32ac6cb 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -39,6 +39,7 @@ import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.utils.Exceptions; @@ -151,7 +152,10 @@ public static TreeMap searchForConnectors(String connectorsDi for (Path archive : stream) { try { - NarClassLoader ncl = NarClassLoader.getFromArchive(new File(archive.toString()), Collections.emptySet(), narExtractionDirectory); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(archive.toString())) + .extractionDirectory(narExtractionDirectory) + .build(); Connector.ConnectorBuilder connectorBuilder = Connector.builder(); ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index acfd00e7a4bce..57a5052eda700 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -28,23 +28,14 @@ import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarUnpacker; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.config.validation.ConfigValidationAnnotations; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.testng.PowerMockTestCase; import org.testng.annotations.Test; -import java.io.File; import java.io.IOException; import java.lang.reflect.Field; -import java.nio.file.Files; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -53,18 +44,14 @@ import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE; import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE; import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.mockito.ArgumentMatchers.any; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; /** - * Unit test of {@link Reflections}. + * Unit test of {@link SinkConfigUtilsTest}. */ -@PrepareForTest({ConnectorUtils.class, NarUnpacker.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "javax.xml.*", "org.xml.*", "org.w3c.dom.*", "org.springframework.context.*", "org.apache.log4j.*", "com.sun.org.apache.xerces.*", "javax.management.*" }) public class SinkConfigUtilsTest extends PowerMockTestCase { private ConnectorDefinition defn; @@ -430,29 +417,16 @@ public void testMergeRuntimeFlags() { @Test public void testValidateConfig() throws IOException { - mockStatic(ConnectorUtils.class); - mockStatic(NarUnpacker.class); - defn = new ConnectorDefinition(); - defn.setSinkConfigClass(TestSinkConfig.class.getName()); - PowerMockito.when(ConnectorUtils.getConnectorDefinition(any())).thenReturn(defn); - - File tmpdir = Files.createTempDirectory("test").toFile(); - PowerMockito.when(NarUnpacker.unpackNar(any(), any())).thenReturn(tmpdir); - SinkConfig sinkConfig = createSinkConfig(); - NarClassLoader narClassLoader = NarClassLoader.getFromArchive( - tmpdir, Collections.emptySet(), - Thread.currentThread().getContextClassLoader(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); - // Good config sinkConfig.getConfigs().put("configParameter", "Test"); - SinkConfigUtils.validateSinkConfig(sinkConfig, narClassLoader); + SinkConfigUtils.validateSinkConfig(sinkConfig, TestSinkConfig.class); // Bad config sinkConfig.getConfigs().put("configParameter", null); Exception e = expectThrows(IllegalArgumentException.class, - () -> SinkConfigUtils.validateSinkConfig(sinkConfig, narClassLoader)); + () -> SinkConfigUtils.validateSinkConfig(sinkConfig, TestSinkConfig.class)); assertTrue(e.getMessage().contains("Could not validate sink config: Field 'configParameter' cannot be null!")); } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java index 22b5afa452c99..a63cf14bc3d57 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java @@ -28,41 +28,27 @@ import org.apache.pulsar.common.io.BatchSourceConfig; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SourceConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarUnpacker; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.config.validation.ConfigValidationAnnotations; import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.BatchSourceTriggerer; import org.apache.pulsar.io.core.SourceContext; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.testng.PowerMockTestCase; import org.testng.annotations.Test; -import java.io.File; import java.io.IOException; import java.lang.reflect.Field; -import java.nio.file.Files; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.mockito.ArgumentMatchers.any; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; /** - * Unit test of {@link Reflections}. + * Unit test of {@link SourceConfigUtilsTest}. */ -@PrepareForTest({ConnectorUtils.class, NarUnpacker.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "javax.xml.*", "org.xml.*", "org.w3c.dom.*", "org.springframework.context.*", "org.apache.log4j.*", "com.sun.org.apache.xerces.*", "javax.management.*" }) public class SourceConfigUtilsTest extends PowerMockTestCase { private ConnectorDefinition defn; @@ -305,29 +291,16 @@ public void testMergeDifferentBatchSourceConfig() { @Test public void testValidateConfig() throws IOException { - mockStatic(ConnectorUtils.class); - mockStatic(NarUnpacker.class); - defn = new ConnectorDefinition(); - defn.setSourceConfigClass(SourceConfigUtilsTest.TestSourceConfig.class.getName()); - PowerMockito.when(ConnectorUtils.getConnectorDefinition(any())).thenReturn(defn); - - File tmpdir = Files.createTempDirectory("test").toFile(); - PowerMockito.when(NarUnpacker.unpackNar(any(), any())).thenReturn(tmpdir); - SourceConfig sourceConfig = createSourceConfig(); - NarClassLoader narClassLoader = NarClassLoader.getFromArchive( - tmpdir, Collections.emptySet(), - Thread.currentThread().getContextClassLoader(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); - // Good config sourceConfig.getConfigs().put("configParameter", "Test"); - SourceConfigUtils.validateSourceConfig(sourceConfig, narClassLoader); + SourceConfigUtils.validateSourceConfig(sourceConfig, SourceConfigUtilsTest.TestSourceConfig.class); // Bad config sourceConfig.getConfigs().put("configParameter", null); Exception e = expectThrows(IllegalArgumentException.class, - () -> SourceConfigUtils.validateSourceConfig(sourceConfig, narClassLoader)); + () -> SourceConfigUtils.validateSourceConfig(sourceConfig, SourceConfigUtilsTest.TestSourceConfig.class)); assertTrue(e.getMessage().contains("Could not validate source config: Field 'configParameter' cannot be null!")); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index d37049fe4029e..9d1d1233b3024 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -448,7 +448,8 @@ public synchronized void restartFunctionInstances(String tenant, String namespac /** * Restart the entire function or restart a single instance of the function */ - private void restartFunctionUsingPulsarAdmin(Assignment assignment, String tenant, String namespace, + @VisibleForTesting + void restartFunctionUsingPulsarAdmin(Assignment assignment, String tenant, String namespace, String functionName, boolean restartEntireFunction) throws PulsarAdminException { ComponentType componentType = assignment.getInstance().getFunctionMetaData().getFunctionDetails().getComponentType(); if (restartEntireFunction) { @@ -501,7 +502,8 @@ public void stopAllOwnedFunctions() { } } - private void stopFunction(String fullyQualifiedInstanceId, boolean restart) throws Exception { + @VisibleForTesting + void stopFunction(String fullyQualifiedInstanceId, boolean restart) throws Exception { log.info("[{}] {}..", restart ? "restarting" : "stopping", fullyQualifiedInstanceId); FunctionRuntimeInfo functionRuntimeInfo = this.getFunctionRuntimeInfo(fullyQualifiedInstanceId); if (functionRuntimeInfo != null) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceLoader.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceLoader.java index 877f6129cfcad..20e6340e6d973 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceLoader.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/WorkerServiceLoader.java @@ -24,10 +24,10 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; @@ -50,8 +50,10 @@ public class WorkerServiceLoader { */ public static WorkerServiceDefinition getWorkerServiceDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive( - new File(narPath), Collections.emptySet(), narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build();) { return getWorkerServiceDefinition(ncl); } } @@ -72,10 +74,12 @@ private static WorkerServiceDefinition getWorkerServiceDefinition(NarClassLoader */ static WorkerServiceWithClassLoader load(WorkerServiceMetadata metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - WorkerService.class.getClassLoader(), narExtractionDirectory); + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(WorkerService.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); WorkerServiceDefinition phDef = getWorkerServiceDefinition(ncl); if (StringUtils.isBlank(phDef.getHandlerClass())) { diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinatorTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinatorTest.java index d0f612d48b2c3..6362fa9e8a7fe 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinatorTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinatorTest.java @@ -33,27 +33,16 @@ import java.util.function.Supplier; import org.apache.pulsar.functions.worker.executor.MockExecutorController; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; +import org.powermock.reflect.Whitebox; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link ClusterServiceCoordinator}. */ -@PrepareForTest({ ClusterServiceCoordinator.class }) -@PowerMockIgnore({ "javax.management.*", "org.apache.logging.log4j.*" }) public class ClusterServiceCoordinatorTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private LeaderService leaderService; private ClusterServiceCoordinator coordinator; private ScheduledExecutorService mockExecutor; @@ -62,20 +51,15 @@ public IObjectFactory getObjectFactory() { @BeforeMethod public void setup() throws Exception { - PowerMockito.mockStatic(Executors.class); - this.mockExecutor = PowerMockito.mock(ScheduledExecutorService.class); + this.mockExecutor = mock(ScheduledExecutorService.class); this.mockExecutorController = new MockExecutorController() .controlScheduleAtFixedRate(mockExecutor, 10); - PowerMockito.when( - Executors.newSingleThreadScheduledExecutor( - any(ThreadFactory.class)) - ).thenReturn(mockExecutor); - this.leaderService = mock(LeaderService.class); this.checkIsStillLeader = () -> leaderService.isLeader(); this.coordinator = new ClusterServiceCoordinator("test-coordinator", leaderService, checkIsStillLeader); + Whitebox.setInternalState(coordinator, "executor", mockExecutor); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index be832236aacc2..26cebc3bf8dc5 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -31,7 +31,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -68,13 +67,12 @@ import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.utils.FunctionCommon; import org.mockito.ArgumentMatchers; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; import java.util.HashMap; @@ -84,22 +82,10 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; -@PrepareForTest({FunctionRuntimeManager.class, RuntimeFactory.class}) @Slf4j -@PowerMockIgnore({ - "javax.management.*", - "javax.ws.*", - "org.apache.logging.log4j.*", - "org.apache.pulsar.functions.runtime.thread" -}) public class FunctionRuntimeManagerTest { private final String PULSAR_SERVICE_URL = "pulsar://localhost:6650"; - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testProcessAssignmentUpdateAddFunctions() throws Exception { @@ -124,71 +110,70 @@ public void testProcessAssignmentUpdateAddFunctions() throws Exception { PulsarWorkerService workerService = mock(PulsarWorkerService.class); doReturn(pulsarClient).when(workerService).getClient(); doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin(); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - - // test new assignment add functions - FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class))); - FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); - - Function.Assignment assignment1 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - Function.Assignment assignment2 = Function.Assignment.newBuilder() - .setWorkerId("worker-2") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function2).setInstanceId(0).build()) - .build(); - - List assignments = new LinkedList<>(); - assignments.add(assignment1); - assignments.add(assignment2); - - functionRuntimeManager.processAssignment(assignment1); - functionRuntimeManager.processAssignment(assignment2); - - verify(functionRuntimeManager, times(2)).setAssignment(any(Function.Assignment.class)); - verify(functionRuntimeManager, times(0)).deleteAssignment(any(Function.Assignment.class)); - assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 2); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); - assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-2") - .get("test-tenant/test-namespace/func-2:0"), assignment2); - verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner).startFunction(argThat( - functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function1))); - verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); - assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), - new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build())); + // test new assignment add functions + FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class))); + FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); + + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); + + Function.Assignment assignment1 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + Function.Assignment assignment2 = Function.Assignment.newBuilder() + .setWorkerId("worker-2") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function2).setInstanceId(0).build()) + .build(); + + List assignments = new LinkedList<>(); + assignments.add(assignment1); + assignments.add(assignment2); + + functionRuntimeManager.processAssignment(assignment1); + functionRuntimeManager.processAssignment(assignment2); + + verify(functionRuntimeManager, times(2)).setAssignment(any(Function.Assignment.class)); + verify(functionRuntimeManager, times(0)).deleteAssignment(any(Function.Assignment.class)); + assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 2); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); + assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-2") + .get("test-tenant/test-namespace/func-2:0"), assignment2); + verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner).startFunction(argThat( + functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function1))); + verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); + assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), + new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build())); + } } @Test @@ -214,74 +199,80 @@ public void testProcessAssignmentUpdateDeleteFunctions() throws Exception { doReturn(pulsarClient).when(workerService).getClient(); doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin(); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - - // test new assignment delete functions - FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class))); - FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); - - // Delete this assignment - Function.Assignment assignment1 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - Function.Assignment assignment2 = Function.Assignment.newBuilder() - .setWorkerId("worker-2") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function2).setInstanceId(0).build()) - .build(); - - // add existing assignments - functionRuntimeManager.setAssignment(assignment1); - functionRuntimeManager.setAssignment(assignment2); - reset(functionRuntimeManager); - - functionRuntimeManager.functionRuntimeInfos.put( - "test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build())); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); - functionRuntimeManager.processAssignment(assignment1); - functionRuntimeManager.processAssignment(assignment2); - functionRuntimeManager.deleteAssignment(FunctionCommon.getFullyQualifiedInstanceId(assignment1.getInstance())); - verify(functionRuntimeManager, times(0)).setAssignment(any(Function.Assignment.class)); - verify(functionRuntimeManager, times(1)).deleteAssignment(any(String.class)); - - assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-2").get("test-tenant/test-namespace/func-2:0"), assignment2); - - verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(1)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner).terminateFunction(argThat( - functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function1))); + // test new assignment delete functions + FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class))); + FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); + + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); + + // Delete this assignment + Function.Assignment assignment1 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + Function.Assignment assignment2 = Function.Assignment.newBuilder() + .setWorkerId("worker-2") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function2).setInstanceId(0).build()) + .build(); + + // add existing assignments + functionRuntimeManager.setAssignment(assignment1); + functionRuntimeManager.setAssignment(assignment2); + reset(functionRuntimeManager); + + functionRuntimeManager.functionRuntimeInfos.put( + "test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build())); + + functionRuntimeManager.processAssignment(assignment1); + functionRuntimeManager.processAssignment(assignment2); + + functionRuntimeManager.deleteAssignment(FunctionCommon.getFullyQualifiedInstanceId(assignment1.getInstance())); + verify(functionRuntimeManager, times(0)).setAssignment(any(Function.Assignment.class)); + verify(functionRuntimeManager, times(1)).deleteAssignment(any(String.class)); + + assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-2").get("test-tenant/test-namespace/func-2:0"), assignment2); + + verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(1)).terminateFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner).terminateFunction(argThat( + functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function1))); + + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0); + } + } - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0); + private void mockRuntimeFactory(MockedStatic runtimeFactoryMockedStatic) { + runtimeFactoryMockedStatic.when(() -> RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) + .thenAnswer((Answer) invocation -> new ThreadRuntimeFactory()); } @Test @@ -306,117 +297,116 @@ public void testProcessAssignmentUpdateModifyFunctions() throws Exception { doReturn(pulsarClient).when(workerService).getClient(); doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin(); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - - // test new assignment update functions - FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class)); - FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); - - Function.Assignment assignment1 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - Function.Assignment assignment2 = Function.Assignment.newBuilder() - .setWorkerId("worker-2") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function2).setInstanceId(0).build()) - .build(); - - // add existing assignments - functionRuntimeManager.setAssignment(assignment1); - functionRuntimeManager.setAssignment(assignment2); - reset(functionActioner); - - Function.Assignment assignment3 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function2).setInstanceId(0).build()) - .build(); - - functionRuntimeManager.functionRuntimeInfos.put( - "test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build())); - functionRuntimeManager.functionRuntimeInfos.put( - "test-tenant/test-namespace/func-2:0", new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0) - .build())); - - functionRuntimeManager.processAssignment(assignment1); - functionRuntimeManager.processAssignment(assignment3); - - verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); - // make sure terminate is not called since this is a update operation - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - - verify(functionActioner).stopFunction(argThat( - functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); - - verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner).startFunction(argThat( - functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); - - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2); - assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment3); - - reset(functionActioner); - - // add a stop - Function.FunctionMetaData.Builder function2StoppedBldr = function2.toBuilder(); - function2StoppedBldr.putInstanceStates(0, Function.FunctionState.STOPPED); - Function.FunctionMetaData function2Stopped = function2StoppedBldr.build(); - - Function.Assignment assignment4 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function2Stopped).setInstanceId(0).build()) - .build(); - - functionRuntimeManager.processAssignment(assignment4); - - verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); - // make sure terminate is not called since this is a update operation - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - - verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> - functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); - - verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2); - assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment4); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); + // test new assignment update functions + FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class)); + FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); + + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); + + Function.Assignment assignment1 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + Function.Assignment assignment2 = Function.Assignment.newBuilder() + .setWorkerId("worker-2") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function2).setInstanceId(0).build()) + .build(); + + // add existing assignments + functionRuntimeManager.setAssignment(assignment1); + functionRuntimeManager.setAssignment(assignment2); + reset(functionActioner); + + Function.Assignment assignment3 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function2).setInstanceId(0).build()) + .build(); + + functionRuntimeManager.functionRuntimeInfos.put( + "test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build())); + functionRuntimeManager.functionRuntimeInfos.put( + "test-tenant/test-namespace/func-2:0", new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0) + .build())); + + functionRuntimeManager.processAssignment(assignment1); + functionRuntimeManager.processAssignment(assignment3); + + verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); + // make sure terminate is not called since this is a update operation + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + + verify(functionActioner).stopFunction(argThat( + functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); + + verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner).startFunction(argThat( + functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); + + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2); + assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment3); + + reset(functionActioner); + + // add a stop + Function.FunctionMetaData.Builder function2StoppedBldr = function2.toBuilder(); + function2StoppedBldr.putInstanceStates(0, Function.FunctionState.STOPPED); + Function.FunctionMetaData function2Stopped = function2StoppedBldr.build(); + + Function.Assignment assignment4 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function2Stopped).setInstanceId(0).build()) + .build(); + + functionRuntimeManager.processAssignment(assignment4); + + verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); + // make sure terminate is not called since this is a update operation + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + + verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> + functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2))); + + verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2); + assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1); + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment4); + } } @@ -442,93 +432,93 @@ public void testReassignment() throws Exception { doReturn(pulsarClient).when(workerService).getClient(); doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin(); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - - // test new assignment update functions - FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class)); - FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - - - Function.Assignment assignment1 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); - /** Test transfer from me to other worker **/ - - // add existing assignments - functionRuntimeManager.setAssignment(assignment1); - - // new assignment with different worker - Function.Assignment assignment2 = Function.Assignment.newBuilder() - .setWorkerId("worker-2") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - - FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build()); - functionRuntimeManager.functionRuntimeInfos.put( - "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); - - functionRuntimeManager.processAssignment(assignment2); - - verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-2").get("test-tenant/test-namespace/func-1:0"), assignment2); - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0); - assertNull(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0")); - - /** Test transfer from other worker to me **/ - reset(functionActioner); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.Assignment assignment3 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - - functionRuntimeManager.processAssignment(assignment3); - - verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment3); - assertNull(functionRuntimeManager.workerIdToAssignments - .get("worker-2")); - - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); - assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), functionRuntimeInfo); + // test new assignment update functions + FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class)); + FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); + + + Function.Assignment assignment1 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + /** Test transfer from me to other worker **/ + + // add existing assignments + functionRuntimeManager.setAssignment(assignment1); + + // new assignment with different worker + Function.Assignment assignment2 = Function.Assignment.newBuilder() + .setWorkerId("worker-2") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build()); + functionRuntimeManager.functionRuntimeInfos.put( + "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); + + functionRuntimeManager.processAssignment(assignment2); + + verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-2").get("test-tenant/test-namespace/func-1:0"), assignment2); + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0); + assertNull(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0")); + + /** Test transfer from other worker to me **/ + reset(functionActioner); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.Assignment assignment3 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + functionRuntimeManager.processAssignment(assignment3); + + verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment3); + assertNull(functionRuntimeManager.workerIdToAssignments + .get("worker-2")); + + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); + assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), functionRuntimeInfo); + } } @Test @@ -634,46 +624,46 @@ public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { ErrorNotifier errorNotifier = mock(ErrorNotifier.class); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); - // test new assignment add functions - FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - errorNotifier); - FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); - doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); - doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); + // test new assignment add functions + FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + errorNotifier); + FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner()); + doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class)); + doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); - assertEquals(functionRuntimeManager.initialize(), messageId3); + assertEquals(functionRuntimeManager.initialize(), messageId3); - assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); - verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); + assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1); + verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class)); - // verify stop function is called zero times because we don't want to unnecessarily restart any functions during initialization - verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + // verify stop function is called zero times because we don't want to unnecessarily restart any functions during initialization + verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner).startFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().equals(assignment1.getInstance()))); + verify(functionActioner).startFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().equals(assignment1.getInstance()))); - assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); - assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), - new FunctionRuntimeInfo().setFunctionInstance( - Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build())); + assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1); + assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), + new FunctionRuntimeInfo().setFunctionInstance( + Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) + .build())); - // verify no errors occurred - verify(errorNotifier, times(0)).triggerError(any()); + // verify no errors occurred + verify(errorNotifier, times(0)).triggerError(any()); + } } @Test @@ -719,105 +709,107 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { workerConfig, kubernetesRuntimeFactory, null, null, null, null)); - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(anyString())) - .thenReturn(kubernetesRuntimeFactory); - - // test new assignment update functions - FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class)); - functionRuntimeManager.setFunctionActioner(functionActioner); - - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("path").build()) - .setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - - - Function.Assignment assignment1 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - - /** Test transfer from me to other worker **/ - - // add existing assignments - functionRuntimeManager.setAssignment(assignment1); - - // new assignment with different worker - Function.Assignment assignment2 = Function.Assignment.newBuilder() - .setWorkerId("worker-2") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - - Function.Instance instance = Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build(); - FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo() - .setFunctionInstance(instance) - .setRuntimeSpawner(functionActioner.getRuntimeSpawner(instance, function1.getPackageLocation().getPackagePath())); - functionRuntimeManager.functionRuntimeInfos.put( - "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); - - functionRuntimeManager.processAssignment(assignment2); - - // make sure nothing is called - verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-2").get("test-tenant/test-namespace/func-1:0"), assignment2); - assertNull(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0")); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + runtimeFactoryMockedStatic.when(() -> RuntimeFactory.getFuntionRuntimeFactory(anyString())) + .thenAnswer(invocation -> kubernetesRuntimeFactory); - /** Test transfer from other worker to me **/ - Function.Assignment assignment3 = Function.Assignment.newBuilder() - .setWorkerId("worker-1") - .setInstance(Function.Instance.newBuilder() - .setFunctionMetaData(function1).setInstanceId(0).build()) - .build(); - - functionRuntimeManager.processAssignment(assignment3); - - // make sure nothing is called - verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); - verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); - - assertEquals(functionRuntimeManager.workerIdToAssignments - .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment3); - assertNull(functionRuntimeManager.workerIdToAssignments - .get("worker-2")); - - assertEquals( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getFunctionInstance(), - functionRuntimeInfo.getFunctionInstance()); - assertNotNull( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner()); - - assertEquals( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getInstanceConfig().getFunctionDetails(), - function1.getFunctionDetails()); - assertEquals( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getInstanceConfig().getInstanceId(), - instance.getInstanceId()); - assertTrue( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getRuntimeFactory() instanceof KubernetesRuntimeFactory); - assertNotNull( - functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getRuntime()); - - verify(kubernetesRuntime, times(1)).reinitialize(); + // test new assignment update functions + FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class)); + functionRuntimeManager.setFunctionActioner(functionActioner); + + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("path").build()) + .setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); + + + Function.Assignment assignment1 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + /** Test transfer from me to other worker **/ + + // add existing assignments + functionRuntimeManager.setAssignment(assignment1); + + // new assignment with different worker + Function.Assignment assignment2 = Function.Assignment.newBuilder() + .setWorkerId("worker-2") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + Function.Instance instance = Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build(); + FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo() + .setFunctionInstance(instance) + .setRuntimeSpawner(functionActioner.getRuntimeSpawner(instance, function1.getPackageLocation().getPackagePath())); + functionRuntimeManager.functionRuntimeInfos.put( + "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); + + functionRuntimeManager.processAssignment(assignment2); + + // make sure nothing is called + verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-2").get("test-tenant/test-namespace/func-1:0"), assignment2); + assertNull(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0")); + + /** Test transfer from other worker to me **/ + + Function.Assignment assignment3 = Function.Assignment.newBuilder() + .setWorkerId("worker-1") + .setInstance(Function.Instance.newBuilder() + .setFunctionMetaData(function1).setInstanceId(0).build()) + .build(); + + functionRuntimeManager.processAssignment(assignment3); + + // make sure nothing is called + verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class)); + verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class)); + + assertEquals(functionRuntimeManager.workerIdToAssignments + .get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment3); + assertNull(functionRuntimeManager.workerIdToAssignments + .get("worker-2")); + + assertEquals( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getFunctionInstance(), + functionRuntimeInfo.getFunctionInstance()); + assertNotNull( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner()); + + assertEquals( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getInstanceConfig().getFunctionDetails(), + function1.getFunctionDetails()); + assertEquals( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getInstanceConfig().getInstanceId(), + instance.getInstanceId()); + assertTrue( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getRuntimeFactory() instanceof KubernetesRuntimeFactory); + assertNotNull( + functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0").getRuntimeSpawner().getRuntime()); + + verify(kubernetesRuntime, times(1)).reinitialize(); + } } @Test @@ -908,23 +900,23 @@ public void testFunctionRuntimeSetCorrectly() { workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL); workerConfig.setStateStorageServiceUrl("foo"); workerConfig.setFunctionAssignmentTopicName("assignments"); - - mockStatic(RuntimeFactory.class); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - - FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( - workerConfig, - mock(PulsarWorkerService.class), - mock(Namespace.class), - mock(MembershipManager.class), - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class)); - - assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ThreadRuntimeFactory.class); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + mockRuntimeFactory(runtimeFactoryMockedStatic); + + + FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager( + workerConfig, + mock(PulsarWorkerService.class), + mock(Namespace.class), + mock(MembershipManager.class), + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class)); + + assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ThreadRuntimeFactory.class); + } } catch (Exception e) { log.error("Failed to initialize the runtime manager : ", e); fail(); @@ -1058,47 +1050,50 @@ public void testThreadFunctionInstancesRestart() throws Exception { doReturn(functions).when(pulsarAdmin).functions(); doReturn(pulsarAdmin).when(workerService).getFunctionAdmin(); - mockStatic(RuntimeFactory.class); - List workerInfos = new LinkedList<>(); - workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0)); - workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0)); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - MembershipManager membershipManager = mock(MembershipManager.class); - doReturn(workerInfos).when(membershipManager).getCurrentMembership(); - - // build three types of FunctionMetaData - Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("function") - .setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build(); - Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("source") - .setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build(); - Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("sink") - .setComponentType(Function.FunctionDetails.ComponentType.SINK)).build(); - - FunctionRuntimeManager functionRuntimeManager = PowerMockito.spy(new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - membershipManager, - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class))); - - // verify restart function/source/sink using different assignment - verifyRestart(functionRuntimeManager, function, "worker-1", false, false); - verifyRestart(functionRuntimeManager, function, "worker-2", false, true); - verifyRestart(functionRuntimeManager, source, "worker-1", false, false); - verifyRestart(functionRuntimeManager, source, "worker-2", false, true); - verifyRestart(functionRuntimeManager, sink, "worker-1", false, false); - verifyRestart(functionRuntimeManager, sink, "worker-2", false, true); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + + mockRuntimeFactory(runtimeFactoryMockedStatic); + + List workerInfos = new LinkedList<>(); + workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0)); + workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0)); + + MembershipManager membershipManager = mock(MembershipManager.class); + doReturn(workerInfos).when(membershipManager).getCurrentMembership(); + + // build three types of FunctionMetaData + Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("function") + .setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build(); + Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("source") + .setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build(); + Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("sink") + .setComponentType(Function.FunctionDetails.ComponentType.SINK)).build(); + + FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + membershipManager, + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class))); + + // verify restart function/source/sink using different assignment + verifyRestart(functionRuntimeManager, function, "worker-1", false, false); + verifyRestart(functionRuntimeManager, function, "worker-2", false, true); + verifyRestart(functionRuntimeManager, source, "worker-1", false, false); + verifyRestart(functionRuntimeManager, source, "worker-2", false, true); + verifyRestart(functionRuntimeManager, sink, "worker-1", false, false); + verifyRestart(functionRuntimeManager, sink, "worker-2", false, true); + } } @Test @@ -1112,74 +1107,79 @@ public void testKubernetesFunctionInstancesRestart() throws Exception { WorkerConfig.KubernetesContainerFactory kubernetesContainerFactory = new WorkerConfig.KubernetesContainerFactory(); workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory); - KubernetesRuntimeFactory mockedKubernetesRuntimeFactory = spy(KubernetesRuntimeFactory.class); - doNothing().when(mockedKubernetesRuntimeFactory).initialize( - any(WorkerConfig.class), - any(AuthenticationConfig.class), - any(SecretsProviderConfigurator.class), - any(), - any(), - any() - ); - doNothing().when(mockedKubernetesRuntimeFactory).setupClient(); - doReturn(true).when(mockedKubernetesRuntimeFactory).externallyManaged(); - PowerMockito.whenNew(KubernetesRuntimeFactory.class) - .withNoArguments().thenReturn(mockedKubernetesRuntimeFactory); - - PulsarWorkerService workerService = mock(PulsarWorkerService.class); - // mock pulsarAdmin sources sinks functions - PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class); - Sources sources = mock(Sources.class); - doNothing().when(sources).restartSource(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); - doReturn(sources).when(pulsarAdmin).sources(); - Sinks sinks = mock(Sinks.class); - doReturn(sinks).when(pulsarAdmin).sinks(); - Functions functions = mock(Functions.class); - doNothing().when(functions).restartFunction(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); - doReturn(functions).when(pulsarAdmin).functions(); - - doReturn(pulsarAdmin).when(workerService).getFunctionAdmin(); - mockStatic(RuntimeFactory.class); - List workerInfos = new LinkedList<>(); - workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0)); - workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0)); - PowerMockito.when(RuntimeFactory.getFuntionRuntimeFactory(eq(ThreadRuntimeFactory.class.getName()))) - .thenReturn(new ThreadRuntimeFactory()); - MembershipManager membershipManager = mock(MembershipManager.class); - doReturn(workerInfos).when(membershipManager).getCurrentMembership(); - - // build three types of FunctionMetaData - Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("function") - .setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build(); - Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("source") - .setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build(); - Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails( - Function.FunctionDetails.newBuilder() - .setTenant("test-tenant").setNamespace("test-namespace").setName("sink") - .setComponentType(Function.FunctionDetails.ComponentType.SINK)).build(); - - FunctionRuntimeManager functionRuntimeManager = PowerMockito.spy(new FunctionRuntimeManager( - workerConfig, - workerService, - mock(Namespace.class), - membershipManager, - mock(ConnectorsManager.class), - mock(FunctionsManager.class), - mock(FunctionMetaDataManager.class), - mock(WorkerStatsManager.class), - mock(ErrorNotifier.class))); - - // verify restart function/source/sink using different assignment - verifyRestart(functionRuntimeManager, function, "worker-1",true, false); - verifyRestart(functionRuntimeManager, function, "worker-2", true, true); - verifyRestart(functionRuntimeManager, source, "worker-1", true, false); - verifyRestart(functionRuntimeManager, source, "worker-2", true, true); - verifyRestart(functionRuntimeManager, sink, "worker-1", true, false); - verifyRestart(functionRuntimeManager, sink, "worker-2", true, true); + try (MockedConstruction mocked = Mockito.mockConstruction(KubernetesRuntimeFactory.class, + (mockedKubernetesRuntimeFactory, context) -> { + doNothing().when(mockedKubernetesRuntimeFactory).initialize( + any(WorkerConfig.class), + any(AuthenticationConfig.class), + any(SecretsProviderConfigurator.class), + any(), + any(), + any() + ); + doNothing().when(mockedKubernetesRuntimeFactory).setupClient(); + doReturn(true).when(mockedKubernetesRuntimeFactory).externallyManaged(); + + })) { + + PulsarWorkerService workerService = mock(PulsarWorkerService.class); + // mock pulsarAdmin sources sinks functions + PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class); + Sources sources = mock(Sources.class); + doNothing().when(sources).restartSource(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); + doReturn(sources).when(pulsarAdmin).sources(); + Sinks sinks = mock(Sinks.class); + doReturn(sinks).when(pulsarAdmin).sinks(); + Functions functions = mock(Functions.class); + doNothing().when(functions).restartFunction(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); + doReturn(functions).when(pulsarAdmin).functions(); + + doReturn(pulsarAdmin).when(workerService).getFunctionAdmin(); + try (final MockedStatic runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class);) { + + mockRuntimeFactory(runtimeFactoryMockedStatic); + + List workerInfos = new LinkedList<>(); + workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0)); + workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0)); + + MembershipManager membershipManager = mock(MembershipManager.class); + doReturn(workerInfos).when(membershipManager).getCurrentMembership(); + + // build three types of FunctionMetaData + Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("function") + .setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build(); + Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("source") + .setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build(); + Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() + .setTenant("test-tenant").setNamespace("test-namespace").setName("sink") + .setComponentType(Function.FunctionDetails.ComponentType.SINK)).build(); + + FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager( + workerConfig, + workerService, + mock(Namespace.class), + membershipManager, + mock(ConnectorsManager.class), + mock(FunctionsManager.class), + mock(FunctionMetaDataManager.class), + mock(WorkerStatsManager.class), + mock(ErrorNotifier.class))); + + // verify restart function/source/sink using different assignment + verifyRestart(functionRuntimeManager, function, "worker-1", true, false); + verifyRestart(functionRuntimeManager, function, "worker-2", true, true); + verifyRestart(functionRuntimeManager, source, "worker-1", true, false); + verifyRestart(functionRuntimeManager, source, "worker-2", true, true); + verifyRestart(functionRuntimeManager, sink, "worker-1", true, false); + verifyRestart(functionRuntimeManager, sink, "worker-2", true, true); + } + } } private static void verifyRestart(FunctionRuntimeManager functionRuntimeManager, Function.FunctionMetaData function, @@ -1193,11 +1193,11 @@ private static void verifyRestart(FunctionRuntimeManager functionRuntimeManager, .findFunctionAssignments("test-tenant", "test-namespace", "function"); functionRuntimeManager.restartFunctionInstances("test-tenant", "test-namespace", "function"); if (expectRestartByPulsarAdmin) { - PowerMockito.verifyPrivate(functionRuntimeManager) - .invoke("restartFunctionUsingPulsarAdmin", assignment, "test-tenant", "test-namespace", "function", externallyManaged); + verify(functionRuntimeManager, times(1)) + .restartFunctionUsingPulsarAdmin(eq(assignment), eq("test-tenant"), + eq("test-namespace"), eq("function"), eq(externallyManaged)); } else { - PowerMockito.verifyPrivate(functionRuntimeManager) - .invoke("stopFunction", FunctionCommon.getFullyQualifiedInstanceId(assignment.getInstance()), true); + verify(functionRuntimeManager).stopFunction(eq(FunctionCommon.getFullyQualifiedInstanceId(assignment.getInstance())), eq(true)); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java index e2bb78484f84b..759d1504258ed 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java @@ -74,20 +74,12 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.doReturn; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; -@PrepareForTest({WorkerUtils.class, InstanceUtils.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*" }) public class FunctionsImplTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final class TestFunction implements org.apache.pulsar.functions.api.Function { @Override @@ -198,8 +190,6 @@ public void setup() throws Exception { this.resource = spy(new FunctionsImpl(() -> mockedWorkerService)); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(any())).thenReturn(Function.FunctionDetails.ComponentType.FUNCTION); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 3c457f0862df2..40fd4ae94ae52 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -19,17 +19,16 @@ package org.apache.pulsar.functions.worker.rest.api.v2; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.apache.pulsar.functions.utils.FunctionCommon.mergeJson; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.doNothing; -import static org.powermock.api.mockito.PowerMockito.doThrow; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import com.google.common.collect.Lists; @@ -43,12 +42,14 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Consumer; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; @@ -76,6 +77,7 @@ import org.apache.pulsar.functions.proto.Function.SubscriptionType; import org.apache.pulsar.functions.runtime.RuntimeFactory; import org.apache.pulsar.functions.source.TopicSchema; +import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; @@ -87,28 +89,18 @@ import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2; import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link FunctionsApiV2Resource}. */ -@PrepareForTest({WorkerUtils.class, InstanceUtils.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.functions.api.*" }) public class FunctionApiV2ResourceTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final class TestFunction implements Function { @Override @@ -147,6 +139,7 @@ public String process(String input, Context context) { private FunctionMetaData mockedFunctionMetadata; private LeaderService mockedLeaderService; private PulsarFunctionTestTemporaryDirectory tempDirectory; + private static Map mockStaticContexts = new HashMap<>(); @BeforeMethod public void setup() throws Exception { @@ -196,9 +189,6 @@ public void setup() throws Exception { FunctionsImpl functions = spy(new FunctionsImpl(() -> mockedWorkerService)); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(any())).thenReturn(FunctionDetails.ComponentType.FUNCTION); - this.resource = spy(new FunctionsImplV2(functions)); } @@ -208,6 +198,34 @@ public void cleanup() { if (tempDirectory != null) { tempDirectory.delete(); } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + private void mockStatic(Class classStatic, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); + consumer.accept(mockedStatic); + } + + private void mockWorkerUtils() { + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + }); + } + private void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + if (consumer != null) { + consumer.accept(ctx); + } + }); + } + + private void mockInstanceUtils() { + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(any())) + .thenReturn(FunctionDetails.ComponentType.FUNCTION); + }); } // @@ -506,17 +524,13 @@ private void testRegisterFunctionMissingArguments( } functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - mockStatic(WorkerUtils.class); - - try { - WorkerUtils.uploadFileToBookkeeper( + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( anyString(), any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); - } catch (Exception e) { - throw new RuntimeException(e); - } + any(Namespace.class))) + .thenCallRealMethod(); + }); try { resource.registerFunction( @@ -567,14 +581,13 @@ public void testRegisterExistedFunction() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testRegisterFunctionUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new IOException("upload failure")); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -588,17 +601,29 @@ public void testRegisterFunctionUploadFailure() throws Exception { @Test public void testRegisterFunctionSuccess() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); + try (MockedStatic mocked = Mockito.mockStatic(WorkerUtils.class)) { + mocked.when(() -> WorkerUtils.uploadToBookKeeper( + any(Namespace.class), + any(InputStream.class), + anyString())).thenAnswer((i) -> null); + mocked.when(() -> WorkerUtils.dumpToTmpFile(any())).thenAnswer(i -> + { + try { + File tmpFile = FunctionCommon.createPkgTempFile(); + tmpFile.deleteOnExit(); + Files.copy((InputStream) i.getArguments()[0], tmpFile.toPath(), REPLACE_EXISTING); + return tmpFile; + } catch (IOException e) { + throw new RuntimeException("Cannot create a temporary file", e); + } + + } + ); + WorkerUtils.uploadToBookKeeper(null, null, null); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + registerDefaultFunction(); + } } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); throw re; @@ -630,13 +655,7 @@ public void testRegisterFunctionNonexistantTenant() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") public void testRegisterFunctionFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); doThrow(new IllegalArgumentException("function failed to register")) @@ -649,16 +668,12 @@ public void testRegisterFunctionFailure() throws Exception { } } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") public void testRegisterFunctionInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -741,10 +756,7 @@ public void testUpdateFunctionMissingFunctionName() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingPackage() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, namespace, @@ -766,10 +778,7 @@ public void testUpdateFunctionMissingPackage() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingInputTopic() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -792,10 +801,7 @@ public void testUpdateFunctionMissingInputTopic() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingClassName() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -818,10 +824,7 @@ public void testUpdateFunctionMissingClassName() throws Exception { @Test public void testUpdateFunctionChangedParallelism() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -843,10 +846,7 @@ public void testUpdateFunctionChangedParallelism() throws Exception { @Test public void testUpdateFunctionChangedInputs() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -865,10 +865,7 @@ public void testUpdateFunctionChangedInputs() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") public void testUpdateFunctionChangedOutput() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); Map someOtherInput = new HashMap<>(); someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); @@ -993,13 +990,13 @@ public void testUpdateNotExistedFunction() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testUpdateFunctionUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new IOException("upload failure")); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1012,13 +1009,7 @@ public void testUpdateFunctionUploadFailure() throws Exception { @Test public void testUpdateFunctionSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1066,13 +1057,7 @@ public void testUpdateFunctionWithUrl() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") public void testUpdateFunctionFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1089,16 +1074,7 @@ public void testUpdateFunctionFailure() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") public void testUpdateFunctionInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); - - + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); doThrow(new IllegalStateException("Function registeration interrupted")) @@ -1313,6 +1289,7 @@ public void testGetNotExistedFunction() throws IOException { @Test public void testGetFunctionSuccess() throws IOException { + mockInstanceUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); SinkSpec sinkSpec = SinkSpec.newBuilder() @@ -1392,6 +1369,7 @@ private List listDefaultFunctions() { @Test public void testListFunctionsSuccess() { + mockInstanceUtils(); final List functions = Lists.newArrayList("test-1", "test-2"); final List metaDataList = new LinkedList<>(); FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index a0c96930bc28b..067ca9e737275 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -22,13 +22,12 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.doNothing; -import static org.powermock.api.mockito.PowerMockito.doThrow; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; @@ -90,29 +89,18 @@ import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.v2.FunctionsApiV2Resource; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link FunctionsApiV2Resource}. */ -@PrepareForTest({WorkerUtils.class, InstanceUtils.class, FunctionUtils.class}) -@PowerMockIgnore({ "javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.functions.api.*" }) public class FunctionApiV3ResourceTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final class TestFunction implements Function { @Override @@ -159,6 +147,7 @@ public void accept(String s) { private LeaderService mockedLeaderService; private Packages mockedPackages; private PulsarFunctionTestTemporaryDirectory tempDirectory; + private static Map mockStaticContexts = new HashMap<>(); @BeforeMethod public void setup() throws Exception { @@ -210,8 +199,6 @@ public void setup() throws Exception { when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); this.resource = spy(new FunctionsImpl(() -> mockedWorkerService)); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(any())).thenReturn(FunctionDetails.ComponentType.FUNCTION); } @AfterMethod(alwaysRun = true) @@ -219,6 +206,33 @@ public void cleanup() { if (tempDirectory != null) { tempDirectory.delete(); } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + private void mockStatic(Class classStatic, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); + consumer.accept(mockedStatic); + } + + private void mockWorkerUtils() { + mockWorkerUtils(null); + } + + private void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + if (consumer != null) { + consumer.accept(ctx); + } + }); + } + + private void mockInstanceUtils() { + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(any())) + .thenReturn(FunctionDetails.ComponentType.FUNCTION); + }); } // @@ -611,14 +625,15 @@ public void testRegisterExistedFunction() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testRegisterFunctionUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + } + ).thenThrow(new IOException("upload failure"));; + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -632,14 +647,7 @@ public void testRegisterFunctionUploadFailure() throws Exception { @Test public void testRegisterFunctionSuccess() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -692,14 +700,7 @@ public void testRegisterFunctionNonexistantTenant() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") public void testRegisterFunctionFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -716,14 +717,7 @@ public void testRegisterFunctionFailure() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") public void testRegisterFunctionInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); @@ -807,10 +801,7 @@ public void testUpdateFunctionMissingFunctionName() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingPackage() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, namespace, @@ -832,10 +823,7 @@ public void testUpdateFunctionMissingPackage() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingInputTopic() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -858,10 +846,7 @@ public void testUpdateFunctionMissingInputTopic() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateFunctionMissingClassName() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -884,10 +869,7 @@ public void testUpdateFunctionMissingClassName() throws Exception { @Test public void testUpdateFunctionChangedParallelism() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -909,10 +891,7 @@ public void testUpdateFunctionChangedParallelism() throws Exception { @Test public void testUpdateFunctionChangedInputs() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateFunctionMissingArguments( tenant, @@ -931,10 +910,7 @@ public void testUpdateFunctionChangedInputs() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") public void testUpdateFunctionChangedOutput() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); Map someOtherInput = new HashMap<>(); someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); @@ -1055,13 +1031,16 @@ public void testUpdateNotExistedFunction() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testUpdateFunctionUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + + }).thenThrow(new IOException("upload failure")); + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1074,13 +1053,7 @@ public void testUpdateFunctionUploadFailure() throws Exception { @Test public void testUpdateFunctionSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1122,13 +1095,7 @@ public void testUpdateFunctionWithUrl() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") public void testUpdateFunctionFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1145,13 +1112,7 @@ public void testUpdateFunctionFailure() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") public void testUpdateFunctionInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString()); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -1386,6 +1347,7 @@ public void testGetNotExistedFunction() { @Test public void testGetFunctionSuccess() { + mockInstanceUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); SinkSpec sinkSpec = SinkSpec.newBuilder() @@ -1465,6 +1427,7 @@ private List listDefaultFunctions() { @Test public void testListFunctionsSuccess() { + mockInstanceUtils(); final List functions = Lists.newArrayList("test-1", "test-2"); final List metaDataList = new LinkedList<>(); FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( @@ -1486,21 +1449,25 @@ public void testOnlyGetSources() { List functions = Lists.newArrayList("test-2"); List functionMetaDataList = new LinkedList<>(); FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build()).build(); + FunctionDetails.newBuilder() + .setName("test-1") + .setComponentType(FunctionDetails.ComponentType.SOURCE) + .build()).build(); functionMetaDataList.add(f1); FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build()).build(); + FunctionDetails.newBuilder() + .setName("test-2") + .setComponentType(FunctionDetails.ComponentType.FUNCTION) + .build()).build(); functionMetaDataList.add(f2); FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-3").build()).build(); + FunctionDetails.newBuilder() + .setName("test-3") + .setComponentType(FunctionDetails.ComponentType.SINK) + .build()).build(); functionMetaDataList.add(f3); when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(f1.getFunctionDetails())).thenReturn(FunctionDetails.ComponentType.SOURCE); - PowerMockito.when(InstanceUtils.calculateSubjectType(f2.getFunctionDetails())).thenReturn(FunctionDetails.ComponentType.FUNCTION); - PowerMockito.when(InstanceUtils.calculateSubjectType(f3.getFunctionDetails())).thenReturn(FunctionDetails.ComponentType.SINK); - List functionList = listDefaultFunctions(); assertEquals(functions, functionList); } @@ -1549,7 +1516,7 @@ public void testDownloadFunctionFile() throws Exception { @Test public void testDownloadFunctionBuiltin() throws Exception { - mockStatic(WorkerUtils.class); + mockStatic(WorkerUtils.class, ctx -> {}); URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); @@ -1573,8 +1540,10 @@ public void testDownloadFunctionBuiltin() throws Exception { mock(org.apache.pulsar.functions.utils.functions.Functions.class); when(mockedFunctions.getFunctions()).thenReturn(functionsMap); - mockStatic(FunctionUtils.class); - PowerMockito.when(FunctionUtils.searchForFunctions(anyString(), anyBoolean())).thenReturn(mockedFunctions); + mockStatic(FunctionUtils.class, ctx -> { + ctx.when(() -> FunctionUtils.searchForFunctions(anyString(), anyBoolean())).thenReturn(mockedFunctions); + + }); StreamingOutput streamOutput = function.downloadFunction("builtin://cassandra", null, null); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index f5f24de32bc6f..f6a6e1fcad1bb 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -23,15 +23,12 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.doNothing; -import static org.powermock.api.mockito.PowerMockito.doReturn; -import static org.powermock.api.mockito.PowerMockito.doThrow; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import com.google.common.collect.Lists; @@ -48,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; @@ -85,32 +83,18 @@ import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SinksImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.testng.Assert; -import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link SinksApiV3Resource}. */ -@PrepareForTest({WorkerUtils.class, SinkConfigUtils.class, ConnectorUtils.class, FunctionCommon.class, - ClassLoaderUtils.class, InstanceUtils.class}) -@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.io.*", - "java.io.*"}) - public class SinkApiV3ResourceTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final String tenant = "test-tenant"; private static final String namespace = "test-namespace"; private static final String sink = "test-sink"; @@ -142,6 +126,7 @@ public IObjectFactory getObjectFactory() { private LeaderService mockedLeaderService; private Packages mockedPackages; private PulsarFunctionTestTemporaryDirectory tempDirectory; + private static Map mockStaticContexts = new HashMap<>(); private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path"; @@ -219,8 +204,7 @@ public void setup() throws Exception { when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); this.resource = spy(new SinksImpl(() -> mockedWorkerService)); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(any())).thenReturn(FunctionDetails.ComponentType.SINK); + } @AfterMethod(alwaysRun = true) @@ -228,8 +212,35 @@ public void cleanup() { if (tempDirectory != null) { tempDirectory.delete(); } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + private void mockStatic(Class classStatic, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); + consumer.accept(mockedStatic); + } + private void mockWorkerUtils() { + mockWorkerUtils(null); + } + + private void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + if (consumer != null) { + consumer.accept(ctx); + } + }); } + private void mockInstanceUtils() { + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(any())) + .thenReturn(FunctionDetails.ComponentType.SINK); + }); + } + + // // Register Functions // @@ -316,6 +327,7 @@ public void testRegisterSinkMissingPackage() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink class UnknownClass must " + "be in class path") public void testRegisterSinkWrongClassName() { + mockInstanceUtils(); try { testRegisterSinkMissingArguments( tenant, @@ -339,6 +351,7 @@ public void testRegisterSinkWrongClassName() { + " or JAR package. Sink classname is not provided and attempts to load it as a NAR package produced the " + "following error.") public void testRegisterSinkMissingPackageDetails() { + mockInstanceUtils(); try { testRegisterSinkMissingArguments( tenant, @@ -360,6 +373,7 @@ public void testRegisterSinkMissingPackageDetails() { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Failed to extract sink class " + "from archive") public void testRegisterSinkInvalidJarNoSink() throws IOException { + mockInstanceUtils(); try { try (FileInputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { testRegisterSinkMissingArguments( @@ -383,6 +397,7 @@ public void testRegisterSinkInvalidJarNoSink() throws IOException { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Must specify at least one " + "topic of input via topicToSerdeClassName, topicsPattern, topicToSchemaType or inputSpecs") public void testRegisterSinkNoInput() throws IOException { + mockInstanceUtils(); try { try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { testRegisterSinkMissingArguments( @@ -406,6 +421,7 @@ public void testRegisterSinkNoInput() throws IOException { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink parallelism must be a " + "positive number") public void testRegisterSinkNegativeParallelism() throws IOException { + mockInstanceUtils(); try { try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { testRegisterSinkMissingArguments( @@ -429,6 +445,7 @@ public void testRegisterSinkNegativeParallelism() throws IOException { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink parallelism must be a " + "positive number") public void testRegisterSinkZeroParallelism() throws IOException { + mockInstanceUtils(); try { try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { testRegisterSinkMissingArguments( @@ -574,15 +591,15 @@ public void testRegisterExistedSink() throws IOException { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testRegisterSinkUploadFailure() throws Exception { + mockInstanceUtils(); try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new IOException("upload failure")); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -595,14 +612,8 @@ public void testRegisterSinkUploadFailure() throws Exception { @Test public void testRegisterSinkSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockInstanceUtils(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -611,14 +622,8 @@ public void testRegisterSinkSuccess() throws Exception { @Test public void testRegisterSinkConflictingFields() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockInstanceUtils(); + mockWorkerUtils(); String actualTenant = "DIFFERENT_TENANT"; String actualNamespace = "DIFFERENT_NAMESPACE"; @@ -650,15 +655,9 @@ public void testRegisterSinkConflictingFields() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "sink failed to register") public void testRegisterSinkFailure() throws Exception { + mockInstanceUtils(); try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -675,15 +674,9 @@ public void testRegisterSinkFailure() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration " + "interrupted") public void testRegisterSinkInterrupted() throws Exception { + mockInstanceUtils(); try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -699,6 +692,7 @@ public void testRegisterSinkInterrupted() throws Exception { @Test(timeOut = 20000) public void testRegisterSinkSuccessWithPackageName() throws IOException { + mockInstanceUtils(); registerDefaultSinkWithPackageUrl("sink://public/default/test@v1"); } @@ -778,11 +772,7 @@ public void testUpdateSinkMissingFunctionName() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateSinkMissingPackage() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSinkMissingArguments( tenant, @@ -803,11 +793,7 @@ public void testUpdateSinkMissingPackage() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateSinkMissingInputs() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSinkMissingArguments( tenant, @@ -828,11 +814,7 @@ public void testUpdateSinkMissingInputs() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") public void testUpdateSinkDifferentInputs() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); Map inputTopics = new HashMap<>(); inputTopics.put("DifferentTopic", DEFAULT_SERDE); @@ -854,11 +836,7 @@ public void testUpdateSinkDifferentInputs() throws Exception { @Test public void testUpdateSinkDifferentParallelism() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSinkMissingArguments( tenant, @@ -882,25 +860,22 @@ private void testUpdateSinkMissingArguments( String className, Integer parallelism, String expectedError) throws Exception { - mockStatic(ConnectorUtils.class); - doReturn(CASSANDRA_STRING_SINK).when(ConnectorUtils.class); - ConnectorUtils.getIOSinkClass(any(NarClassLoader.class)); - - mockStatic(ClassLoaderUtils.class); - - mockStatic(FunctionCommon.class); - PowerMockito.when(FunctionCommon.class, "createPkgTempFile").thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); - - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSinkType(any()); - - doReturn(mock(NarClassLoader.class)).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); - - doReturn(ATLEAST_ONCE).when(FunctionCommon.class); - FunctionCommon.convertProcessingGuarantee(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE); + mockStatic(ConnectorUtils.class, ctx -> { + ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) + .thenReturn(CASSANDRA_STRING_SINK); + }); + + mockStatic(ClassLoaderUtils.class, ctx -> {}); + + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); + ctx.when(() -> FunctionCommon + .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) + .thenReturn(ATLEAST_ONCE); + }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -958,25 +933,22 @@ private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception sinkConfig.setParallelism(parallelism); sinkConfig.setTopicToSerdeClassName(topicsToSerDeClassName); - mockStatic(ConnectorUtils.class); - doReturn(CASSANDRA_STRING_SINK).when(ConnectorUtils.class); - ConnectorUtils.getIOSinkClass(any(NarClassLoader.class)); - - mockStatic(ClassLoaderUtils.class); + mockStatic(ConnectorUtils.class, ctx -> { + ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) + .thenReturn(CASSANDRA_STRING_SINK); + }); - mockStatic(FunctionCommon.class); - PowerMockito.when(FunctionCommon.class, "createPkgTempFile").thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); + mockStatic(ClassLoaderUtils.class, ctx -> {}); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSinkType(any()); - - doReturn(mock(NarClassLoader.class)).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); - - doReturn(ATLEAST_ONCE).when(FunctionCommon.class); - FunctionCommon.convertProcessingGuarantee(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); + ctx.when(() -> FunctionCommon + .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) + .thenReturn(ATLEAST_ONCE); + }); this.mockedFunctionMetaData = @@ -1010,14 +982,14 @@ public void testUpdateNotExistedSink() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testUpdateSinkUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new IOException("upload failure")); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1030,13 +1002,7 @@ public void testUpdateSinkUploadFailure() throws Exception { @Test public void testUpdateSinkSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1058,24 +1024,23 @@ public void testUpdateSinkWithUrl() throws Exception { sinkConfig.setParallelism(parallelism); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - mockStatic(ConnectorUtils.class); - doReturn(CASSANDRA_STRING_SINK).when(ConnectorUtils.class); - ConnectorUtils.getIOSinkClass(any(NarClassLoader.class)); - - mockStatic(ClassLoaderUtils.class); - mockStatic(FunctionCommon.class); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSinkType(any()); - PowerMockito.when(FunctionCommon.class, "extractFileFromPkgURL", any()).thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); + mockStatic(ConnectorUtils.class, ctx -> { + ctx.when(() -> ConnectorUtils.getIOSinkClass(any())) + .thenReturn(CASSANDRA_STRING_SINK); + }); - doReturn(mock(NarClassLoader.class)).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); + mockStatic(ClassLoaderUtils.class, ctx -> {}); - doReturn(ATLEAST_ONCE).when(FunctionCommon.class); - FunctionCommon.convertProcessingGuarantee(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); + ctx.when(() -> FunctionCommon + .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) + .thenReturn(ATLEAST_ONCE); + }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -1095,14 +1060,7 @@ public void testUpdateSinkWithUrl() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "sink failed to register") public void testUpdateSinkFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); - + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); doThrow(new IllegalArgumentException("sink failed to register")) @@ -1138,13 +1096,7 @@ public void testUpdateSinkFailedWithWrongPackageName() throws Exception { + "interrupted") public void testUpdateSinkInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1243,6 +1195,7 @@ public void testDeregisterSinkSuccess() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "sink failed to deregister") public void testDeregisterSinkFailure() throws Exception { + mockInstanceUtils(); try { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1262,6 +1215,7 @@ public void testDeregisterSinkFailure() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregistration " + "interrupted") public void testDeregisterSinkInterrupted() throws Exception { + mockInstanceUtils(); try { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1280,78 +1234,89 @@ public void testDeregisterSinkInterrupted() throws Exception { @Test public void testDeregisterSinkBKPackageCleanup() throws IOException { + mockInstanceUtils(); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - String packagePath = - "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-pulsar-io-batch-data-generator-2.7.0.nar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + String packagePath = + "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-pulsar-io-batch-data-generator-2.7.0.nar"; + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); - deregisterDefaultSink(); + deregisterDefaultSink(); - PowerMockito.verifyStatic(WorkerUtils.class, times(1)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(1)); + } } @Test - public void testDeregisterBuiltinSinkBKPackageCleanup() throws IOException { + public void testDeregisterBuiltinSinkBKPackageCleanup() { + mockInstanceUtils(); - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - String packagePath = String.format("%s://data-generator", Utils.BUILTIN); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + String packagePath = String.format("%s://data-generator", Utils.BUILTIN); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); - deregisterDefaultSink(); + deregisterDefaultSink(); - // if the sink is a builtin sink we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + // if the sink is a builtin sink we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); + } } @Test - public void testDeregisterHTTPSinkBKPackageCleanup() throws IOException { + public void testDeregisterHTTPSinkBKPackageCleanup() { + mockInstanceUtils(); - mockStatic(WorkerUtils.class); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - String packagePath = "http://foo.com/connector.jar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + String packagePath = "http://foo.com/connector.jar"; + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + + deregisterDefaultSink(); - deregisterDefaultSink(); + // if the sink is a is download from a http url, we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); - // if the sink is a is download from a http url, we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + } } @Test public void testDeregisterFileSinkBKPackageCleanup() throws IOException { + mockInstanceUtils(); - mockStatic(WorkerUtils.class); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - String packagePath = "file://foo/connector.jar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + String packagePath = "file://foo/connector.jar"; + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); - deregisterDefaultSink(); + deregisterDefaultSink(); - // if the sink package has a file url, we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + // if the sink package has a file url, we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); + } } // @@ -1525,6 +1490,7 @@ private List listDefaultSinks() { @Test public void testListSinksSuccess() { + mockInstanceUtils(); final List functions = Lists.newArrayList("test-1", "test-2"); final List functionMetaDataList = new LinkedList<>(); functionMetaDataList.add(FunctionMetaData.newBuilder().setFunctionDetails( @@ -1554,13 +1520,15 @@ public void testOnlyGetSinks() { functionMetaDataList.add(f3); when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(f1.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.SOURCE); - PowerMockito.when(InstanceUtils.calculateSubjectType(f2.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - PowerMockito.when(InstanceUtils.calculateSubjectType(f3.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.SINK); + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(eq(f1.getFunctionDetails()))) + .thenReturn(FunctionDetails.ComponentType.SOURCE); + ctx.when(() -> InstanceUtils.calculateSubjectType(eq(f2.getFunctionDetails()))) + .thenReturn(FunctionDetails.ComponentType.FUNCTION); + ctx.when(() -> InstanceUtils.calculateSubjectType(eq(f3.getFunctionDetails()))) + .thenReturn(FunctionDetails.ComponentType.SINK); + + }); List sinkList = listDefaultSinks(); assertEquals(functions, sinkList); @@ -1613,27 +1581,25 @@ private FunctionDetails createDefaultFunctionDetails() throws IOException { public void testRegisterSinkSuccessK8sNoUpload() throws Exception { mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false); - mockStatic(WorkerUtils.class); - doThrow(new RuntimeException("uploadFileToBookkeeper triggered")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered")); - mockStatic(FunctionCommon.class); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSinkType(any()); - PowerMockito.when(FunctionCommon.class, "extractFileFromPkgURL", any()).thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); + }); - doReturn(true).when(FunctionCommon.class); - FunctionCommon.isFunctionCodeBuiltin(any()); + NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); + ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - doReturn(mock(NarClassLoader.class)).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); + }); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); Connector connector = Connector.builder() .classLoader(mockedClassLoader) @@ -1670,27 +1636,24 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { final String injectedErrMsg = "uploadFileToBookkeeper triggered"; mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true); - mockStatic(WorkerUtils.class); - doThrow(new RuntimeException(injectedErrMsg)).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - mockStatic(FunctionCommon.class); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSinkType(any()); - PowerMockito.when(FunctionCommon.class, "extractFileFromPkgURL", any()).thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException(injectedErrMsg)); - doReturn(true).when(FunctionCommon.class); - FunctionCommon.isFunctionCodeBuiltin(any()); - - doReturn(mock(NarClassLoader.class)).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); + }); NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); + ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); + }); + ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); Connector connector = Connector.builder() .classLoader(mockedClassLoader) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index de4c19f810b36..2b41594374485 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -21,19 +21,17 @@ import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOCassandraNar; import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOInvalidNar; import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOTwitterNar; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.doNothing; -import static org.powermock.api.mockito.PowerMockito.doReturn; -import static org.powermock.api.mockito.PowerMockito.doThrow; -import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import com.google.common.collect.Lists; import java.io.File; @@ -44,8 +42,11 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; @@ -59,11 +60,11 @@ import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.api.utils.IdentityFunction; -import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; @@ -84,31 +85,19 @@ import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SourcesImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; /** * Unit test of {@link SourcesApiV3Resource}. */ -@PrepareForTest({WorkerUtils.class, ConnectorUtils.class, FunctionCommon.class, ClassLoaderUtils.class, - InstanceUtils.class}) -@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*", "org.apache.pulsar.io.*"}) public class SourceApiV3ResourceTest { - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - private static final String tenant = "test-tenant"; private static final String namespace = "test-namespace"; private static final String source = "test-source"; @@ -137,10 +126,11 @@ public IObjectFactory getObjectFactory() { private PulsarFunctionTestTemporaryDirectory tempDirectory; private static NarClassLoader narClassLoader; + private static Map mockStaticContexts = new HashMap<>(); @BeforeClass public void setupNarClassLoader() throws IOException { - narClassLoader = NarClassLoader.getFromArchive(getPulsarIOTwitterNar(), Collections.emptySet()); + narClassLoader = NarClassLoaderBuilder.builder().narFile(getPulsarIOTwitterNar()).build(); } @AfterClass(alwaysRun = true) @@ -203,8 +193,11 @@ public void setup() throws Exception { when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); this.resource = spy(new SourcesImpl(() -> mockedWorkerService)); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(any())).thenReturn(FunctionDetails.ComponentType.SOURCE); + } + + private void mockStatic(Class classStatic, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); + consumer.accept(mockedStatic); } @AfterMethod(alwaysRun = true) @@ -212,6 +205,24 @@ public void cleanup() { if (tempDirectory != null) { tempDirectory.delete(); } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + private void mockWorkerUtils() { + mockStatic(WorkerUtils.class, + ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + }); + } + + private void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + if (consumer != null) { + consumer.accept(ctx); + } + }); } // @@ -542,19 +553,22 @@ public void testRegisterExistedSource() throws IOException { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testRegisterSourceUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); + mockWorkerUtils(ctx -> { + ctx.when(() -> + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new IOException("upload failure")); - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); registerDefaultSource(); + fail(); } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); throw re; @@ -563,20 +577,15 @@ public void testRegisterSourceUploadFailure() throws Exception { @Test public void testRegisterSourceSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); registerDefaultSource(); } + + @Test(timeOut = 20000) public void testRegisterSourceSuccessWithPackageName() throws IOException { registerDefaultSourceWithPackageUrl("source://public/default/test@v1"); @@ -597,12 +606,7 @@ public void testRegisterSourceFailedWithWrongPackageName() throws PulsarAdminExc @Test public void testRegisterSourceConflictingFields() throws Exception { - mockStatic(WorkerUtils.class); - PowerMockito.doNothing().when(WorkerUtils.class, "uploadFileToBookkeeper", anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); String actualTenant = "DIFFERENT_TENANT"; String actualNamespace = "DIFFERENT_NAMESPACE"; @@ -636,14 +640,7 @@ public void testRegisterSourceConflictingFields() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "source failed to register") public void testRegisterSourceFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); @@ -661,14 +658,7 @@ public void testRegisterSourceFailure() throws Exception { + "interrupted") public void testRegisterSourceInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); @@ -749,9 +739,7 @@ public void testUpdateSourceMissingFunctionName() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateSourceMissingPackage() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); + mockStatic(WorkerUtils.class, ctx -> {}); testUpdateSourceMissingArguments( tenant, @@ -773,9 +761,7 @@ public void testUpdateSourceMissingPackage() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") public void testUpdateSourceMissingTopicName() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); + mockStatic(WorkerUtils.class, ctx -> {}); testUpdateSourceMissingArguments( tenant, @@ -798,11 +784,7 @@ public void testUpdateSourceMissingTopicName() throws Exception { + "positive number") public void testUpdateSourceNegativeParallelism() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSourceMissingArguments( tenant, @@ -824,11 +806,7 @@ public void testUpdateSourceNegativeParallelism() throws Exception { @Test public void testUpdateSourceChangedParallelism() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSourceMissingArguments( tenant, @@ -849,9 +827,7 @@ public void testUpdateSourceChangedParallelism() throws Exception { @Test public void testUpdateSourceChangedTopic() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); + mockWorkerUtils(); testUpdateSourceMissingArguments( tenant, @@ -870,11 +846,7 @@ public void testUpdateSourceChangedTopic() throws Exception { + "positive number") public void testUpdateSourceZeroParallelism() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.downloadFromBookkeeper(any(Namespace.class), any(File.class), anyString()); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); testUpdateSourceMissingArguments( tenant, @@ -905,22 +877,18 @@ private void testUpdateSourceMissingArguments( Integer parallelism, String expectedError) throws Exception { - mockStatic(ConnectorUtils.class); - doReturn(TWITTER_FIRE_HOSE).when(ConnectorUtils.class); - ConnectorUtils.getIOSourceClass(any(NarClassLoader.class)); + mockStatic(ConnectorUtils.class, c -> {}); + mockStatic(ClassLoaderUtils.class, c -> {}); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) + .thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) + .thenReturn(narClassLoader); - mockStatic(ClassLoaderUtils.class); - mockStatic(FunctionCommon.class); - PowerMockito.when(FunctionCommon.class, "createPkgTempFile").thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); - - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE))); - - doReturn(narClassLoader).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); + }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -982,22 +950,18 @@ private void updateDefaultSourceWithPackageUrl(String packageUrl) throws Excepti sourceConfig.setTopicName(outputTopic); sourceConfig.setSerdeClassName(outputSerdeClassName); - mockStatic(ConnectorUtils.class); - doReturn(TWITTER_FIRE_HOSE).when(ConnectorUtils.class); - ConnectorUtils.getIOSourceClass(any(NarClassLoader.class)); - - mockStatic(ClassLoaderUtils.class); - - mockStatic(FunctionCommon.class); - PowerMockito.when(FunctionCommon.class, "createPkgTempFile").thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); + mockStatic(ConnectorUtils.class, c -> {}); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE))); + mockStatic(ClassLoaderUtils.class, c -> {}); - doReturn(narClassLoader).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(File.class), any()); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) + .thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) + .thenReturn(narClassLoader); + }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -1031,14 +995,13 @@ public void testUpdateNotExistedSource() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") public void testUpdateSourceUploadFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doThrow(new IOException("upload failure")).when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))).thenThrow(new IOException("upload failure")); + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); + }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); updateDefaultSource(); @@ -1050,14 +1013,7 @@ public void testUpdateSourceUploadFailure() throws Exception { @Test public void testUpdateSourceSuccess() throws Exception { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); @@ -1080,21 +1036,17 @@ public void testUpdateSourceWithUrl() throws Exception { sourceConfig.setParallelism(parallelism); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - mockStatic(ConnectorUtils.class); - doReturn(TWITTER_FIRE_HOSE).when(ConnectorUtils.class); - ConnectorUtils.getIOSourceClass(any(NarClassLoader.class)); - - mockStatic(ClassLoaderUtils.class); - - mockStatic(FunctionCommon.class); - doReturn(String.class).when(FunctionCommon.class); - FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE))); - PowerMockito.when(FunctionCommon.class, "extractFileFromPkgURL", any()).thenCallRealMethod(); - PowerMockito.when(FunctionCommon.class, "getClassLoaderFromPackage", any(), any(), any(), any()) - .thenCallRealMethod(); - - doReturn(narClassLoader).when(FunctionCommon.class); - FunctionCommon.extractNarClassLoader(any(), any()); + mockStatic(ConnectorUtils.class, c -> {}); + mockStatic(ClassLoaderUtils.class, c -> {}); + + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) + .thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) + .thenReturn(narClassLoader); + }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -1115,14 +1067,7 @@ public void testUpdateSourceWithUrl() throws Exception { @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "source failed to register") public void testUpdateSourceFailure() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); @@ -1140,14 +1085,7 @@ public void testUpdateSourceFailure() throws Exception { "interrupted") public void testUpdateSourceInterrupted() throws Exception { try { - mockStatic(WorkerUtils.class); - doNothing().when(WorkerUtils.class); - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - PowerMockito.when(WorkerUtils.class, "dumpToTmpFile", any()).thenCallRealMethod(); + mockWorkerUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); @@ -1308,78 +1246,80 @@ public void testDeregisterSourceInterrupted() throws Exception { @Test public void testDeregisterSourceBKPackageCleanup() throws IOException { - - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - String packagePath = "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-pulsar-io-batch-data-generator-2.7.0.nar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - deregisterDefaultSource(); - PowerMockito.verifyStatic(WorkerUtils.class, times(1)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + + deregisterDefaultSource(); + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(1)); + } } @Test public void testDeregisterBuiltinSourceBKPackageCleanup() throws IOException { - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - String packagePath = String.format("%s://data-generator", Utils.BUILTIN); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - deregisterDefaultSource(); - // if the source is a builtin source we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + + deregisterDefaultSource(); + // if the source is a builtin source we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); + } } @Test public void testDeregisterHTTPSourceBKPackageCleanup() throws IOException { - - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - String packagePath = "http://foo.com/connector.jar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - deregisterDefaultSource(); - // if the source is a is download from a http url, we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + + deregisterDefaultSource(); + // if the source is a is download from a http url, we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); + } } @Test public void testDeregisterFileSourceBKPackageCleanup() throws IOException { - mockStatic(WorkerUtils.class); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); + String packagePath = "file://foo/connector.jar"; - String packagePath = "file://foo/connector.jar"; - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { - deregisterDefaultSource(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))) + .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( + PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); - // if the source has a file url, we shouldn't try to clean it up - PowerMockito.verifyStatic(WorkerUtils.class, times(0)); - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + deregisterDefaultSource(); + // if the source has a file url, we shouldn't try to clean it up + ctx.verify(() -> { + WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); + }, times(0)); + } } // @@ -1558,23 +1498,24 @@ public void testOnlyGetSources() { final List functions = Lists.newArrayList("test-1"); final List functionMetaDataList = new LinkedList<>(); FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build()).build(); + FunctionDetails.newBuilder() + .setName("test-1") + .setComponentType(FunctionDetails.ComponentType.SOURCE) + .build()).build(); functionMetaDataList.add(f1); FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build()).build(); + FunctionDetails.newBuilder() + .setName("test-2") + .setComponentType(FunctionDetails.ComponentType.FUNCTION) + .build()).build(); functionMetaDataList.add(f2); FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-3").build()).build(); + FunctionDetails.newBuilder() + .setName("test-3") + .setComponentType(FunctionDetails.ComponentType.SINK) + .build()).build(); functionMetaDataList.add(f3); when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); - mockStatic(InstanceUtils.class); - PowerMockito.when(InstanceUtils.calculateSubjectType(f1.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.SOURCE); - PowerMockito.when(InstanceUtils.calculateSubjectType(f2.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - PowerMockito.when(InstanceUtils.calculateSubjectType(f3.getFunctionDetails())) - .thenReturn(FunctionDetails.ComponentType.SINK); - List sourceList = listDefaultSources(); assertEquals(functions, sourceList); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensionsUtils.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensionsUtils.java index f8a532d694ca2..2acf3da1c44d1 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensionsUtils.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensionsUtils.java @@ -25,11 +25,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; /** @@ -50,8 +50,10 @@ class ProxyExtensionsUtils { */ public static ProxyExtensionDefinition getProxyExtensionDefinition(String narPath, String narExtractionDirectory) throws IOException { - try (NarClassLoader ncl = NarClassLoader.getFromArchive(new File(narPath), Collections.emptySet(), - narExtractionDirectory)) { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build();) { return getProxyExtensionDefinition(ncl); } } @@ -117,10 +119,12 @@ public static ExtensionsDefinitions searchForExtensions(String extensionsDirecto */ static ProxyExtensionWithClassLoader load(ProxyExtensionMetadata metadata, String narExtractionDirectory) throws IOException { - NarClassLoader ncl = NarClassLoader.getFromArchive( - metadata.getArchivePath().toAbsolutePath().toFile(), - Collections.emptySet(), - ProxyExtension.class.getClassLoader(), narExtractionDirectory); + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(ProxyExtension.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); ProxyExtensionDefinition phDef = getProxyExtensionDefinition(ncl); if (StringUtils.isBlank(phDef.getExtensionClass())) { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionUtilsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionUtilsTest.java index 86ae7fffa775e..7f87a9d4649be 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionUtilsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/ProxyExtensionUtilsTest.java @@ -19,41 +19,26 @@ package org.apache.pulsar.proxy.extensions; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.testng.IObjectFactory; -import org.testng.annotations.ObjectFactory; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; - -import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.util.Set; import static org.apache.pulsar.proxy.extensions.ProxyExtensionsUtils.PROXY_EXTENSION_DEFINITION_FILE; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; import static org.testng.AssertJUnit.fail; -@PrepareForTest({ - ProxyExtensionsUtils.class, NarClassLoader.class -}) -@PowerMockIgnore({"org.apache.logging.log4j.*"}) @Test(groups = "broker") public class ProxyExtensionUtilsTest { - // Necessary to make PowerMockito.mockStatic work with TestNG. - @ObjectFactory - public IObjectFactory getObjectFactory() { - return new org.powermock.modules.testng.PowerMockObjectFactory(); - } - @Test public void testLoadProtocolHandler() throws Exception { ProxyExtensionDefinition def = new ProxyExtensionDefinition(); @@ -73,19 +58,18 @@ public void testLoadProtocolHandler() throws Exception { when(mockLoader.loadClass(eq(MockProxyExtension.class.getName()))) .thenReturn(handlerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); - ProxyExtensionWithClassLoader returnedPhWithCL = ProxyExtensionsUtils.load(metadata, ""); - ProxyExtension returnedPh = returnedPhWithCL.getExtension(); - assertSame(mockLoader, returnedPhWithCL.getClassLoader()); - assertTrue(returnedPh instanceof MockProxyExtension); + ProxyExtensionWithClassLoader returnedPhWithCL = ProxyExtensionsUtils.load(metadata, ""); + ProxyExtension returnedPh = returnedPhWithCL.getExtension(); + + assertSame(mockLoader, returnedPhWithCL.getClassLoader()); + assertTrue(returnedPh instanceof MockProxyExtension); + } } @Test @@ -106,19 +90,17 @@ public void testLoadProtocolHandlerBlankHandlerClass() throws Exception { when(mockLoader.loadClass(eq(MockProxyExtension.class.getName()))) .thenReturn(handlerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); - - try { - ProxyExtensionsUtils.load(metadata, ""); - fail("Should not reach here"); - } catch (IOException ioe) { - // expected + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); + + try { + ProxyExtensionsUtils.load(metadata, ""); + fail("Should not reach here"); + } catch (IOException ioe) { + // expected + } } } @@ -141,19 +123,17 @@ public void testLoadProtocolHandlerWrongHandlerClass() throws Exception { when(mockLoader.loadClass(eq(Runnable.class.getName()))) .thenReturn(handlerClass); - PowerMockito.mockStatic(NarClassLoader.class); - PowerMockito.when(NarClassLoader.getFromArchive( - any(File.class), - any(Set.class), - any(ClassLoader.class), - any(String.class) - )).thenReturn(mockLoader); - - try { - ProxyExtensionsUtils.load(metadata, ""); - fail("Should not reach here"); - } catch (IOException ioe) { - // expected + final NarClassLoaderBuilder mockedBuilder = mock(NarClassLoaderBuilder.class, RETURNS_SELF); + when(mockedBuilder.build()).thenReturn(mockLoader); + try (MockedStatic builder = Mockito.mockStatic(NarClassLoaderBuilder.class)) { + builder.when(() -> NarClassLoaderBuilder.builder()).thenReturn(mockedBuilder); + + try { + ProxyExtensionsUtils.load(metadata, ""); + fail("Should not reach here"); + } catch (IOException ioe) { + // expected + } } }