Use Common Sanitizers (#39935)

* target the version of the proxy using the common sanitizers

* Update eng/target_proxy_version.txt

* use newest proxy version

* newest version with fixed token

* update for tests

* revert

* Update target_proxy_version.txt

* Add removeSanitizer API, with default removed list + azure-core-test update

* update core-test dep

* sendRequestWithRetries for playback requests

* added $..name removable sanitizer

* add AZSDK2003 to set of removed sanitizers

* disable specific test sanitizers

* fix appconfig, search and remove disable sanitizer for storage

* mark some search test live only

* fix storage, attestation, resourcemanager, phonenumbers

* resourcemanager-msi and storage-datalake fix

* can we try a version of the proxy that locks the session sanitizers when cloning them?

* enable removesanitizer on record

* Update target_proxy_version.txt

* fix KV, quantum jobs and resourcemanager

* make logs ingestion testUploadFailure LiveOnly

* mark timout resourcemanager test live only

* update monitor ingestion tests

* update monitor hearbeat test init delay

* marking heartBeatPayloadContainsDataByDefault LiveOnly

* disable canSendKnownLengthNoBufferAsyncFluxBinaryData

* fix monitor tests

---------

Co-authored-by: Sameeksha Vaity (from Dev Box) <savaity@microsoft.com>
Co-authored-by: Vinay Gera <vigera@microsoft.com>

eng/target_proxy_version.txt

@@ -1 +1 @@
-1.0.0-dev.20240410.1
+1.0.0-dev.20240514.2

eng/versioning/version_client.txt

@@ -477,6 +477,7 @@ io.clientcore:http-stress;1.0.0-beta.1;1.0.0-beta.1
 # note: The unreleased dependencies will not be manipulated with the automatic PR creation code.
 # In the pom, the version update tag after the version should name the unreleased package and the dependency version:
 # <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
+unreleased_com.azure:azure-core-test;1.26.0-beta.1
 
 # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current
 # version and set the version to the released beta. Released beta dependencies are only valid

sdk/advisor/azure-resourcemanager-advisor/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/agrifood/azure-resourcemanager-agrifood/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/agrifood/azure-verticals-agrifood-farming/pom.xml

@@ -78,7 +78,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/anomalydetector/azure-ai-anomalydetector/pom.xml

@@ -75,7 +75,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/apicenter/azure-resourcemanager-apicenter/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/apimanagement/azure-resourcemanager-apimanagement/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/appcomplianceautomation/azure-resourcemanager-appcomplianceautomation/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/appconfiguration/azure-data-appconfiguration/pom.xml

@@ -67,7 +67,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/appconfiguration/azure-data-appconfiguration/src/test/java/com/azure/data/appconfiguration/AadCredentialTest.java

@@ -15,6 +15,8 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import java.util.Arrays;
+
 import static com.azure.data.appconfiguration.ConfigurationClientTestBase.FAKE_CONNECTION_STRING;
 import static com.azure.data.appconfiguration.TestHelper.DISPLAY_NAME_WITH_ARGUMENTS;
 
@@ -52,8 +54,14 @@ private void setup(HttpClient httpClient, ConfigurationServiceVersion serviceVer
                 builder.addPolicy(interceptorManager.getRecordPolicy()); // Record
             }
 
+
             client = builder.buildClient();
         }
+
+        // Disable `("$.key")` sanitizer
+        if (!interceptorManager.isLiveMode()) {
+            interceptorManager.removeSanitizers(Arrays.asList("AZSDK3447"));
+        }
     }
 
     ConfigurationClientBuilder setHttpClient(HttpClient httpClient, ConfigurationClientBuilder builder) {

sdk/appconfiguration/azure-data-appconfiguration/src/test/java/com/azure/data/appconfiguration/ConfigurationAsyncClientTest.java

@@ -86,7 +86,12 @@ private ConfigurationAsyncClient getConfigurationAsyncClient(HttpClient httpClie
                 builder.addPolicy(interceptorManager.getRecordPolicy());
             } else if (interceptorManager.isPlaybackMode()) {
                 interceptorManager.addMatchers(Collections.singletonList(
-                    new CustomMatcher().setHeadersKeyOnlyMatch(Collections.singletonList("Sync-Token"))));
+                    new CustomMatcher().setHeadersKeyOnlyMatch(Arrays.asList("Sync-Token", "If-Match"))));
+            }
+
+            // Disable `$.key` snanitizer
+            if (!interceptorManager.isLiveMode()) {
+                interceptorManager.removeSanitizers(Arrays.asList("AZSDK3447"));
             }
             return builder.buildAsyncClient();
         });

sdk/appconfiguration/azure-data-appconfiguration/src/test/java/com/azure/data/appconfiguration/ConfigurationClientBuilderTest.java

@@ -199,6 +199,10 @@ public void nullServiceVersion(HttpClient httpClient) {
                 .httpClient(httpClient)
                 .addPolicy(interceptorManager.getRecordPolicy());
         }
+        // Disable `("$.key")` sanitizer
+        if (!interceptorManager.isLiveMode()) {
+            interceptorManager.removeSanitizers(Collections.singletonList("AZSDK3447"));
+        }
 
         ConfigurationSetting addedSetting = clientBuilder.buildClient().setConfigurationSetting(key, null, value);
         assertEquals(addedSetting.getKey(), key);
@@ -229,6 +233,11 @@ public void defaultPipeline() {
             clientBuilder.httpClient(interceptorManager.getPlaybackClient());
         }
 
+        // Disable `("$.key")` sanitizer
+        if (!interceptorManager.isLiveMode()) {
+            interceptorManager.removeSanitizers(Collections.singletonList("AZSDK3447"));
+        }
+
         ConfigurationSetting addedSetting = clientBuilder
             .buildClient()
             .setConfigurationSetting(key, null, value);

sdk/appconfiguration/azure-data-appconfiguration/src/test/java/com/azure/data/appconfiguration/ConfigurationClientTest.java

@@ -83,7 +83,12 @@ private ConfigurationClient getConfigurationClient(HttpClient httpClient,
                 builder.addPolicy(interceptorManager.getRecordPolicy());
             } else if (interceptorManager.isPlaybackMode()) {
                 interceptorManager.addMatchers(Collections.singletonList(
-                    new CustomMatcher().setHeadersKeyOnlyMatch(Collections.singletonList("Sync-Token"))));
+                    new CustomMatcher().setHeadersKeyOnlyMatch(Arrays.asList("Sync-Token", "If-Match"))));
+            }
+
+            // Disable `$.key` snanitizer
+            if (!interceptorManager.isLiveMode()) {
+                interceptorManager.removeSanitizers(Arrays.asList("AZSDK3447"));
             }
             return builder.buildClient();
         });

sdk/appconfiguration/azure-resourcemanager-appconfiguration/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/appcontainers/azure-resourcemanager-appcontainers/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/applicationinsights/azure-resourcemanager-applicationinsights/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/astro/azure-resourcemanager-astro/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/attestation/azure-resourcemanager-attestation/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/attestation/azure-security-attestation/pom.xml

@@ -65,7 +65,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/attestation/azure-security-attestation/src/test/java/com/azure/security/attestation/AttestationPolicyManagementTests.java

@@ -4,6 +4,7 @@
 
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.rest.Response;
+import com.azure.core.test.annotation.LiveOnly;
 import com.azure.core.util.Context;
 import com.azure.security.attestation.models.AttestationSigner;
 import com.azure.security.attestation.models.AttestationSignerCollection;
@@ -26,7 +27,9 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
+@LiveOnly
 public class AttestationPolicyManagementTests extends AttestationClientTestBase {
+    // LiveOnly because "JWT cannot be stored in recordings."
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("getAttestationClients")
     void testGetPolicyManagementCertificates(HttpClient httpClient, String clientUri) {

sdk/attestation/azure-security-attestation/src/test/java/com/azure/security/attestation/AttestationPolicyTests.java

@@ -4,6 +4,7 @@
 
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.rest.Response;
+import com.azure.core.test.annotation.LiveOnly;
 import com.azure.core.util.Base64Util;
 import com.azure.core.util.BinaryData;
 import com.azure.core.util.Context;
@@ -30,7 +31,9 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
+@LiveOnly
 public class AttestationPolicyTests extends AttestationClientTestBase {
+    // LiveOnly because "JWT cannot be stored in recordings."
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("getPolicyClients")
     void testGetAttestationPolicy(HttpClient client, String clientUri, AttestationType attestationType) {
@@ -372,6 +375,7 @@ void testSetAttestationPolicyAsync(HttpClient httpClient, String clientUri, Atte
         }
     }
 
+    @LiveOnly
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("getPolicyClients")
     void testSetAttestationPolicyAadAsync(HttpClient httpClient, String clientUri, AttestationType attestationType) {
@@ -397,6 +401,7 @@ void testSetAttestationPolicyAadAsync(HttpClient httpClient, String clientUri, A
             .verify();
     }
 
+    @LiveOnly
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("getPolicyClients")
     void testSetAttestationPolicyWithResponseAsync(HttpClient httpClient, String clientUri,

sdk/attestation/azure-security-attestation/src/test/java/com/azure/security/attestation/AttestationTest.java

@@ -6,6 +6,7 @@
 import com.azure.core.http.HttpHeaders;
 import com.azure.core.http.rest.Response;
 import com.azure.core.test.TestMode;
+import com.azure.core.test.annotation.LiveOnly;
 import com.azure.core.test.utils.TestUtils;
 import com.azure.core.util.BinaryData;
 import com.azure.core.util.Context;
@@ -46,7 +47,9 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
+@LiveOnly
 public class AttestationTest extends AttestationClientTestBase {
+    // LiveOnly because "JWT cannot be stored in recordings."
     private static final String RUNTIME_DATA = "CiAgICAgICAgewogICAgICAgICAgICAiandrIiA6IHsKICAgICAgICAgICAgICAgICJrdHk"
         + "iOiJFQyIsCiAgICAgICAgICAgICAgICAidXNlIjoic2lnIiwKICAgICAgICAgICAgICAgICJjcnYiOiJQLTI1NiIsCiAgICAgICAgICAgICA"
         + "gICAieCI6IjE4d0hMZUlnVzl3Vk42VkQxVHhncHF5MkxzellrTWY2SjhualZBaWJ2aE0iLAogICAgICAgICAgICAgICAgInkiOiJjVjRkUzR"

sdk/avs/azure-resourcemanager-avs/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/azurearcdata/azure-resourcemanager-azurearcdata/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/azurestack/azure-resourcemanager-azurestack/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/azurestackhci/azure-resourcemanager-azurestackhci/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/baremetalinfrastructure/azure-resourcemanager-baremetalinfrastructure/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/batch/azure-resourcemanager-batch/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/billing/azure-resourcemanager-billing/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/billingbenefits/azure-resourcemanager-billingbenefits/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/changeanalysis/azure-resourcemanager-changeanalysis/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/chaos/azure-resourcemanager-chaos/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/cognitiveservices/azure-resourcemanager-cognitiveservices/pom.xml

@@ -61,7 +61,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/commerce/azure-resourcemanager-commerce/pom.xml

@@ -54,7 +54,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/communication/azure-communication-callautomation/pom.xml

@@ -77,7 +77,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.25.0</version> <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.26.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>