[ContractKit] Add retry and backoff around fetch metadata (#5699)

### Description

Add retry and backoff around fetching metadata. Seen a few occasions where the services hosting validator metadata will return transient errors, such as connection resets, in which case an attestation attempt from Valora or a status check from the metadata crawler will fail and not retry. 

### Tested

Locally

### Backwards compatibility

Yes

packages/base/src/async.ts

@@ -1,3 +1,5 @@
+import { Logger } from './logger'
+
 const TAG = 'utils/src/async'
 
 /** Sleep for a number of milliseconds */
@@ -13,7 +15,8 @@ export const retryAsync = async <T extends any[], U>(
   inFunction: InFunction<T, U>,
   tries: number,
   params: T,
-  delay = 100
+  delay = 100,
+  logger: Logger | null = null
 ) => {
   let saveError
   for (let i = 0; i < tries; i++) {
@@ -23,7 +26,9 @@ export const retryAsync = async <T extends any[], U>(
     } catch (error) {
       await sleep(delay)
       saveError = error
-      console.info(`${TAG}/@retryAsync, Failed to execute function on try #${i}:`, error)
+      if (logger) {
+        logger(`${TAG}/@retryAsync, Failed to execute function on try #${i}:`, error)
+      }
     }
   }
 
@@ -37,7 +42,8 @@ export const retryAsyncWithBackOff = async <T extends any[], U>(
   tries: number,
   params: T,
   delay = 100,
-  factor = 1.5
+  factor = 1.5,
+  logger: Logger | null = null
 ) => {
   let saveError
   for (let i = 0; i < tries; i++) {
@@ -47,7 +53,9 @@ export const retryAsyncWithBackOff = async <T extends any[], U>(
     } catch (error) {
       await sleep(Math.pow(factor, i) * delay)
       saveError = error
-      console.info(`${TAG}/@retryAsync, Failed to execute function on try #${i}`, error)
+      if (logger) {
+        logger(`${TAG}/@retryAsync, Failed to execute function on try #${i}`, error)
+      }
     }
   }
 
@@ -63,7 +71,8 @@ export const selectiveRetryAsyncWithBackOff = async <T extends any[], U>(
   dontRetry: string[],
   params: T,
   delay = 100,
-  factor = 1.5
+  factor = 1.5,
+  logger: Logger | null = null
 ) => {
   let saveError
   for (let i = 0; i < tries; i++) {
@@ -75,7 +84,9 @@ export const selectiveRetryAsyncWithBackOff = async <T extends any[], U>(
         throw error
       }
       saveError = error
-      console.info(`${TAG}/@retryAsync, Failed to execute function on try #${i}`, error)
+      if (logger) {
+        logger(`${TAG}/@retryAsync, Failed to execute function on try #${i}`, error)
+      }
     }
     if (i < tries - 1) {
       await sleep(Math.pow(factor, i) * delay)

packages/contractkit/src/identity/claims/verify.ts

@@ -16,12 +16,12 @@ import { ClaimTypes } from './types'
  * @param address The address that is making the claim
  * @returns If valid, returns undefined. If invalid or unable to verify, returns a string with the error
  */
-export async function verifyClaim(kit: ContractKit, claim: Claim, address: string) {
+export async function verifyClaim(kit: ContractKit, claim: Claim, address: string, tries = 3) {
   switch (claim.type) {
     case ClaimTypes.KEYBASE:
       return verifyKeybaseClaim(kit, claim, address)
     case ClaimTypes.ACCOUNT:
-      return verifyAccountClaim(kit, claim, address)
+      return verifyAccountClaim(kit, claim, address, tries)
     case ClaimTypes.DOMAIN:
       return verifyDomainRecord(kit, claim, address)
     default:
@@ -33,7 +33,8 @@ export async function verifyClaim(kit: ContractKit, claim: Claim, address: strin
 export const verifyAccountClaim = async (
   kit: ContractKit,
   claim: AccountClaim,
-  address: string
+  address: string,
+  tries = 3
 ) => {
   const metadataURL = await (await kit.contracts.getAccounts()).getMetadataURL(claim.address)
 
@@ -43,7 +44,7 @@ export const verifyAccountClaim = async (
 
   let metadata: IdentityMetadataWrapper
   try {
-    metadata = await IdentityMetadataWrapper.fetchFromURL(kit, metadataURL)
+    metadata = await IdentityMetadataWrapper.fetchFromURL(kit, metadataURL, tries)
   } catch (error) {
     return `Metadata could not be fetched for ${
       claim.address

packages/contractkit/src/identity/metadata.ts

@@ -1,4 +1,5 @@
 import { Address, eqAddress } from '@celo/base/lib/address'
+import { selectiveRetryAsyncWithBackOff } from '@celo/base/lib/async'
 import { Signer } from '@celo/base/lib/signatureUtils'
 import { AddressType, SignatureType } from '@celo/utils/lib/io'
 import { guessSigner, verifySignature } from '@celo/utils/lib/signatureUtils'
@@ -37,12 +38,19 @@ export class IdentityMetadataWrapper {
     })
   }
 
-  static async fetchFromURL(kit: ContractKit, url: string) {
-    const resp = await fetch(url)
-    if (!resp.ok) {
-      throw new Error(`Request failed with status ${resp.status}`)
-    }
-    return this.fromRawString(kit, await resp.text())
+  static async fetchFromURL(kit: ContractKit, url: string, tries = 3) {
+    return selectiveRetryAsyncWithBackOff(
+      async () => {
+        const resp = await fetch(url)
+        if (!resp.ok) {
+          throw new Error(`Request failed with status ${resp.status}`)
+        }
+        return this.fromRawString(kit, await resp.text())
+      },
+      tries,
+      ['Request failed with status 404'],
+      []
+    )
   }
 
   static fromFile(kit: ContractKit, path: string) {

packages/contractkit/src/wrappers/Attestations.ts

@@ -270,7 +270,8 @@ export class AttestationsWrapper extends BaseWrapper<Attestations> {
    */
   async getActionableAttestations(
     identifier: string,
-    account: Address
+    account: Address,
+    tries = 3
   ): Promise<ActionableAttestation[]> {
     const result = await this.contract.methods
       .getCompletableAttestations(identifier, account)
@@ -279,7 +280,7 @@ export class AttestationsWrapper extends BaseWrapper<Attestations> {
     const results = await concurrentMap(
       5,
       parseGetCompletableAttestations(result),
-      this.isIssuerRunningAttestationService
+      this.makeIsIssuerRunningAttestationService(tries)
     )
 
     return results.map((_) => (_.isValid ? _.result : null)).filter(notEmpty)
@@ -290,48 +291,58 @@ export class AttestationsWrapper extends BaseWrapper<Attestations> {
    * @param identifier Attestation identifier (e.g. phone hash)
    * @param account Address of the account
    */
-  async getNonCompliantIssuers(identifier: string, account: Address): Promise<Address[]> {
+  async getNonCompliantIssuers(
+    identifier: string,
+    account: Address,
+    tries = 3
+  ): Promise<Address[]> {
     const result = await this.contract.methods
       .getCompletableAttestations(identifier, account)
       .call()
 
     const withAttestationServiceURLs = await concurrentMap(
       5,
       parseGetCompletableAttestations(result),
-      this.isIssuerRunningAttestationService
+      this.makeIsIssuerRunningAttestationService(tries)
     )
 
     return withAttestationServiceURLs.map((_) => (_.isValid ? null : _.issuer)).filter(notEmpty)
   }
 
-  private isIssuerRunningAttestationService = async (arg: {
-    blockNumber: number
-    issuer: string
-    metadataURL: string
-  }): Promise<AttestationServiceRunningCheckResult> => {
-    try {
-      const metadata = await IdentityMetadataWrapper.fetchFromURL(this.kit, arg.metadataURL)
-      const attestationServiceURLClaim = metadata.findClaim(ClaimTypes.ATTESTATION_SERVICE_URL)
-
-      if (attestationServiceURLClaim === undefined) {
-        throw new Error(`No attestation service URL registered for ${arg.issuer}`)
-      }
+  private makeIsIssuerRunningAttestationService = (tries = 3) => {
+    return async (arg: {
+      blockNumber: number
+      issuer: string
+      metadataURL: string
+    }): Promise<AttestationServiceRunningCheckResult> => {
+      try {
+        const metadata = await IdentityMetadataWrapper.fetchFromURL(
+          this.kit,
+          arg.metadataURL,
+          tries
+        )
+        const attestationServiceURLClaim = metadata.findClaim(ClaimTypes.ATTESTATION_SERVICE_URL)
+
+        if (attestationServiceURLClaim === undefined) {
+          throw new Error(`No attestation service URL registered for ${arg.issuer}`)
+        }
 
-      const nameClaim = metadata.findClaim(ClaimTypes.NAME)
-
-      // TODO: Once we have status indicators, we should check if service is up
-      // https://github.com/celo-org/celo-monorepo/issues/1586
-      return {
-        isValid: true,
-        result: {
-          blockNumber: arg.blockNumber,
-          issuer: arg.issuer,
-          attestationServiceURL: attestationServiceURLClaim.url,
-          name: nameClaim ? nameClaim.name : undefined,
-        },
+        const nameClaim = metadata.findClaim(ClaimTypes.NAME)
+
+        // TODO: Once we have status indicators, we should check if service is up
+        // https://github.com/celo-org/celo-monorepo/issues/1586
+        return {
+          isValid: true,
+          result: {
+            blockNumber: arg.blockNumber,
+            issuer: arg.issuer,
+            attestationServiceURL: attestationServiceURLClaim.url,
+            name: nameClaim ? nameClaim.name : undefined,
+          },
+        }
+      } catch (error) {
+        return { isValid: false, issuer: arg.issuer }
       }
-    } catch (error) {
-      return { isValid: false, issuer: arg.issuer }
     }
   }