From a0c02f5a39c092a869f4459052aac58ea272ffe3 Mon Sep 17 00:00:00 2001 From: Chen Cen Date: Thu, 18 Aug 2022 11:32:05 -0700 Subject: [PATCH 1/5] remove Flex OP dependency --- identity/build.gradle | 2 - .../android/identity/ml/IDDetectorAnalyzer.kt | 101 +++++++++++------- 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/identity/build.gradle b/identity/build.gradle index e049abcb343..6140b3c73e5 100644 --- a/identity/build.gradle +++ b/identity/build.gradle @@ -25,8 +25,6 @@ dependencies { // vanilla tflite library implementation "org.tensorflow:tensorflow-lite:2.9.0" - // flex ops - implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:2.9.0' // support library to reshape image to the input model shape implementation 'org.tensorflow:tensorflow-lite-support:0.4.1' diff --git a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt index 72c948d3902..a8f7c477e49 100644 --- a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt +++ b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt @@ -51,15 +51,15 @@ internal class IDDetectorAnalyzer( ImageProcessor.Builder().add( ResizeOp(INPUT_HEIGHT, INPUT_WIDTH, ResizeOp.ResizeMethod.BILINEAR) ).add( - NormalizeOp(NORMALIZE_MEAN, NORMALIZE_STD) // normalize to (-1, 1) + NormalizeOp(NORMALIZE_MEAN, NORMALIZE_STD) // normalize to [0, 1) ).build() // add normalization tensorImage = imageProcessor.process(tensorImage) preprocessStat.trackResult() val inferenceStat = modelPerformanceTracker.trackInference() - // inference - input: (1, 224, 224, 3), output: (1, 4), (1, 5) - val boundingBoxes = Array(1) { FloatArray(OUTPUT_BOUNDING_BOX_TENSOR_SIZE) } - val categories = Array(1) { FloatArray(OUTPUT_CATEGORY_TENSOR_SIZE) } + // inference - input: (1, 224, 224, 3), output: (392, 4), (392, 4) + val boundingBoxes = Array(OUTPUT_SIZE) { FloatArray(OUTPUT_BOUNDING_BOX_TENSOR_SIZE) } + val categories = Array(OUTPUT_SIZE) { FloatArray(OUTPUT_CATEGORY_TENSOR_SIZE) } tfliteInterpreter.runForMultipleInputsOutputs( arrayOf(tensorImage.buffer), mapOf( @@ -69,31 +69,43 @@ internal class IDDetectorAnalyzer( ) inferenceStat.trackResult() - // find the category with highest score and build output - val resultIndex = requireNotNull(categories[0].indices.maxByOrNull { categories[0][it] }) - val resultCategory: Category - val resultScore: Float + // To get more results, run nonMaxSuppressionMultiClass on the categories. + // Fut for IDDetector, we just need to find the highest score and return it's + // corresponding box. + var bestIndex = 0 + var bestScore = Float.MIN_VALUE + var bestCategoryIndex = INDEX_INVALID + + // Find the best score in the output 2d array of (392, 4), + // Return its index within range [0, 392) on 1d as bestIndex. + for (currentOutputIndex in 0 until OUTPUT_SIZE) { + val currentScores = categories[currentOutputIndex] // array of 4 + val currentBestCategoryIndex = currentScores.indices.maxBy { + currentScores[it] + } + val currentBestScore = currentScores[currentBestCategoryIndex] + if (bestScore < currentBestScore && currentBestScore > idDetectorMinScore) { + bestScore = currentBestScore + bestIndex = currentOutputIndex + bestCategoryIndex = currentBestCategoryIndex + } - // TODO(ccen) use idDetectorMinScore when server updates the value - if (categories[0][resultIndex] > THRESHOLD) { - resultCategory = requireNotNull(INDEX_CATEGORY_MAP[resultIndex]) - resultScore = categories[0][resultIndex] - } else { - resultCategory = Category.NO_ID - resultScore = 0f } - + val bestCategory = INDEX_CATEGORY_MAP[bestCategoryIndex] ?: Category.INVALID + val bestBoundingBox = boundingBoxes[bestIndex] return IDDetectorOutput( BoundingBox( - left = boundingBoxes[0][0], - top = boundingBoxes[0][1], - width = boundingBoxes[0][2], - height = boundingBoxes[0][3] + bestBoundingBox[0], + bestBoundingBox[1], + bestBoundingBox[2], + bestBoundingBox[3] ), - resultCategory, - resultScore, - categories[0].map { it.roundToMaxDecimals(2) } + bestCategory, + bestScore, + LIST_OF_INDICES.map { + categories[it][bestCategoryIndex].roundToMaxDecimals(2) + } ) } @@ -105,34 +117,45 @@ internal class IDDetectorAnalyzer( private val idDetectorMinScore: Float, private val modelPerformanceTracker: ModelPerformanceTracker ) : AnalyzerFactory< - AnalyzerInput, - IdentityScanState, - AnalyzerOutput, - Analyzer - > { + AnalyzerInput, + IdentityScanState, + AnalyzerOutput, + Analyzer + > { override suspend fun newInstance(): Analyzer { - return IDDetectorAnalyzer(modelFile, idDetectorMinScore, modelPerformanceTracker) + return IDDetectorAnalyzer( + modelFile, + idDetectorMinScore, + modelPerformanceTracker + ) } } internal companion object { + const val OUTPUT_SIZE = 392 + const val INPUT_WIDTH = 224 const val INPUT_HEIGHT = 224 - const val NORMALIZE_MEAN = 127.5f - const val NORMALIZE_STD = 127.5f - const val THRESHOLD = 0.4f + + // (0, 1) + const val NORMALIZE_MEAN = 0f + const val NORMALIZE_STD = 255f const val OUTPUT_BOUNDING_BOX_TENSOR_INDEX = 0 const val OUTPUT_CATEGORY_TENSOR_INDEX = 1 const val OUTPUT_BOUNDING_BOX_TENSOR_SIZE = 4 - private const val INDEX_NO_ID = 0 - const val INDEX_PASSPORT = 1 - const val INDEX_ID_FRONT = 2 - const val INDEX_ID_BACK = 3 - const val INDEX_INVALID = 4 + const val INDEX_PASSPORT = 0 + const val INDEX_ID_FRONT = 1 + const val INDEX_ID_BACK = 2 + const val INDEX_INVALID = 3 + val LIST_OF_INDICES = listOf( + INDEX_PASSPORT, + INDEX_ID_FRONT, + INDEX_ID_BACK, + INDEX_INVALID + ) val INPUT_TENSOR_TYPE: DataType = DataType.FLOAT32 - val OUTPUT_CATEGORY_TENSOR_SIZE = Category.values().size + val OUTPUT_CATEGORY_TENSOR_SIZE = Category.values().size - 1 // no NO_ID val INDEX_CATEGORY_MAP = mapOf( - INDEX_NO_ID to Category.NO_ID, INDEX_PASSPORT to Category.PASSPORT, INDEX_ID_FRONT to Category.ID_FRONT, INDEX_ID_BACK to Category.ID_BACK, From 93ebd3f77a889e0284cd08ca29312f0db7336106 Mon Sep 17 00:00:00 2001 From: Chen Cen Date: Thu, 18 Aug 2022 11:52:57 -0700 Subject: [PATCH 2/5] update scores --- .../java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt index a8f7c477e49..e4ad1bb71b2 100644 --- a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt +++ b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt @@ -80,7 +80,7 @@ internal class IDDetectorAnalyzer( // Find the best score in the output 2d array of (392, 4), // Return its index within range [0, 392) on 1d as bestIndex. for (currentOutputIndex in 0 until OUTPUT_SIZE) { - val currentScores = categories[currentOutputIndex] // array of 4 + val currentScores = categories[currentOutputIndex] val currentBestCategoryIndex = currentScores.indices.maxBy { currentScores[it] } @@ -104,7 +104,7 @@ internal class IDDetectorAnalyzer( bestCategory, bestScore, LIST_OF_INDICES.map { - categories[it][bestCategoryIndex].roundToMaxDecimals(2) + categories[bestIndex][it].roundToMaxDecimals(2) } ) } From c0fcd0afd034bf9948452707493e4cc7692b195f Mon Sep 17 00:00:00 2001 From: Chen Cen Date: Thu, 18 Aug 2022 11:54:16 -0700 Subject: [PATCH 3/5] update comments --- .../java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt index e4ad1bb71b2..ed54474e20e 100644 --- a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt +++ b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt @@ -78,7 +78,7 @@ internal class IDDetectorAnalyzer( var bestCategoryIndex = INDEX_INVALID // Find the best score in the output 2d array of (392, 4), - // Return its index within range [0, 392) on 1d as bestIndex. + // return its index within range [0, 392) on 1d as bestIndex. for (currentOutputIndex in 0 until OUTPUT_SIZE) { val currentScores = categories[currentOutputIndex] val currentBestCategoryIndex = currentScores.indices.maxBy { From af32f5264c2f29c383344ab865a12025ead9529d Mon Sep 17 00:00:00 2001 From: Chen Cen Date: Thu, 18 Aug 2022 12:01:25 -0700 Subject: [PATCH 4/5] lint --- .../stripe/android/identity/ml/IDDetectorAnalyzer.kt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt index ed54474e20e..8fdf7b617ec 100644 --- a/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt +++ b/identity/src/main/java/com/stripe/android/identity/ml/IDDetectorAnalyzer.kt @@ -69,7 +69,6 @@ internal class IDDetectorAnalyzer( ) inferenceStat.trackResult() - // To get more results, run nonMaxSuppressionMultiClass on the categories. // Fut for IDDetector, we just need to find the highest score and return it's // corresponding box. @@ -90,7 +89,6 @@ internal class IDDetectorAnalyzer( bestIndex = currentOutputIndex bestCategoryIndex = currentBestCategoryIndex } - } val bestCategory = INDEX_CATEGORY_MAP[bestCategoryIndex] ?: Category.INVALID val bestBoundingBox = boundingBoxes[bestIndex] @@ -117,11 +115,11 @@ internal class IDDetectorAnalyzer( private val idDetectorMinScore: Float, private val modelPerformanceTracker: ModelPerformanceTracker ) : AnalyzerFactory< - AnalyzerInput, - IdentityScanState, - AnalyzerOutput, - Analyzer - > { + AnalyzerInput, + IdentityScanState, + AnalyzerOutput, + Analyzer + > { override suspend fun newInstance(): Analyzer { return IDDetectorAnalyzer( modelFile, From d7520e8c689ad35f115c1870ea273a102b3ed92b Mon Sep 17 00:00:00 2001 From: Chen Cen Date: Thu, 18 Aug 2022 12:05:10 -0700 Subject: [PATCH 5/5] add changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28e5683eb70..201761bc5a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ * [FIXED][5422](https://github.com/stripe/stripe-android/pull/5422) Card expiration dates with a single-digit month are now preserved correctly when closing and re-opening the `PaymentSheet` via the `FlowController`. +### Identity +* [FIXED][5404](https://github.com/stripe/stripe-android/pull/5404) Remove Flex OP dependency from + Identity SDK and reduce its binary size. + ## 20.9.0 - 2022-08-16 This release contains several bug fixes for Payments, PaymentSheet and Financial Connections. Adds `IdentityVerificationSheet#rememberIdentityVerificationSheet` for Identity.