Skip to content

Commit 49f2994

Browse files
nicolas-raoulnicolas-raoul
authored
Merge pull request #7 from commons-app/progress
Add Progress listener for the chunks processed by the detectors
2 parents 116cdea + 49a8053 commit 49f2994

6 files changed

Lines changed: 163 additions & 54 deletions

File tree

‎demo/src/main/java/org/commons/ml/demo/MainActivity.kt‎

Lines changed: 51 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.commons.ml.demo
22

3+
import android.app.AlertDialog
34
import org.commons.ml.common.DetectionOptions
45
import org.commons.ml.common.DetectionResult
56
import org.commons.ml.common.DetectionType
@@ -13,7 +14,10 @@ import android.graphics.RectF
1314
import android.graphics.drawable.ColorDrawable
1415
import android.net.Uri
1516
import android.os.Bundle
17+
import android.view.LayoutInflater
1618
import android.widget.ImageView
19+
import android.widget.ProgressBar
20+
import android.widget.TextView
1721
import androidx.activity.ComponentActivity
1822
import androidx.activity.compose.setContent
1923
import androidx.activity.result.contract.ActivityResultContracts
@@ -40,9 +44,11 @@ import androidx.compose.ui.unit.dp
4044
import androidx.compose.ui.viewinterop.AndroidView
4145
import androidx.lifecycle.lifecycleScope
4246
import kotlinx.coroutines.Dispatchers
47+
import kotlinx.coroutines.delay
4348
import kotlinx.coroutines.launch
4449
import kotlinx.coroutines.withContext
4550
import java.util.Locale
51+
import kotlin.time.Duration.Companion.milliseconds
4652

4753
/** Standalone demo for local face and license-plate detection. */
4854
class MainActivity : ComponentActivity() {
@@ -54,41 +60,11 @@ class MainActivity : ComponentActivity() {
5460
private var thresholdState by mutableFloatStateOf(0.5f)
5561
private var statusMessage by mutableStateOf("")
5662

57-
private val createRedactedImage =
58-
registerForActivityResult(ActivityResultContracts.CreateDocument("image/jpeg")) { uri ->
59-
val source = sourceUri
60-
val regions = overlay.getDetections()
61-
if (uri == null || source == null || regions.isEmpty()) return@registerForActivityResult
62-
lifecycleScope.launch {
63-
val result = withContext(Dispatchers.IO) {
64-
runCatching {
65-
Ajpegtran.pixelize(
66-
this@MainActivity,
67-
source,
68-
uri,
69-
regions.map {
70-
val bounds = RectF(it.bounds).apply {
71-
inset(-width() * 0.12f, -height() * 0.12f)
72-
}
73-
Ajpegtran.PixelizeRegion(
74-
bounds.left.toInt().coerceAtLeast(0),
75-
bounds.top.toInt().coerceAtLeast(0),
76-
bounds.width().toInt().coerceAtLeast(1),
77-
bounds.height().toInt().coerceAtLeast(1)
78-
)
79-
}
80-
).getOrThrow()
81-
}
82-
}
83-
result.onSuccess { setStatus("Saved ajpegtran-redacted JPEG.") }
84-
.onFailure { setStatus("ajpegtran failed: ${diagnosticMessage(it)}") }
85-
}
63+
private val openImage =
64+
registerForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
65+
uri?.let { loadImage(it) }
8666
}
8767

88-
private val openImage = registerForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
89-
uri?.let { loadImage(it) }
90-
}
91-
9268
override fun onCreate(savedInstanceState: Bundle?) {
9369
super.onCreate(savedInstanceState)
9470
buildUi()
@@ -159,14 +135,44 @@ class MainActivity : ComponentActivity() {
159135
}
160136
setStatus("Running ONNX Runtime locally…")
161137
lifecycleScope.launch {
138+
val dialogView = LayoutInflater.from(this@MainActivity)
139+
.inflate(R.layout.dialog, null)
140+
val titleView = dialogView.findViewById<TextView>(R.id.autodetect_title)
141+
val descView = dialogView.findViewById<TextView>(R.id.autodetect_description)
142+
val percentView = dialogView.findViewById<TextView>(R.id.autodetect_percent)
143+
val progressBar = dialogView.findViewById<ProgressBar>(R.id.autodetect_progress)
144+
145+
titleView?.text = "Auto-blurring faces and license plates."
146+
descView?.text = "You can discard false positives using the ❌ button of each blurring area."
147+
percentView?.text = "0%"
148+
progressBar?.progress = 0
149+
150+
val progressDialog = AlertDialog.Builder(this@MainActivity)
151+
.setView(dialogView)
152+
.setCancelable(false)
153+
.create()
154+
progressDialog.show()
155+
162156
runCatching {
163157
withContext(Dispatchers.Default) {
164158
val started = System.nanoTime()
165159
val options = DetectionOptions(confidenceThreshold = thresholdState)
166-
val result = getDetector().detect(source, options)
160+
val result = getDetector().detect(source, options) { progress ->
161+
launch(Dispatchers.Main) {
162+
val percent = (progress * 100).toInt()
163+
progressBar?.progress = percent
164+
percentView?.text = "$percent%"
165+
}
166+
}
167167
Pair(result, (System.nanoTime() - started) / 1_000_000)
168168
}
169169
}.onSuccess { result ->
170+
// Ensure 100% is displayed and give a brief delay so the user can comfortably read it
171+
progressBar?.progress = 100
172+
percentView?.text = "100%"
173+
delay(700.milliseconds)
174+
progressDialog.dismiss()
175+
170176
val detections = when (val value = result.first) {
171177
is DetectionResult.Success -> value.detections
172178
is DetectionResult.Partial -> value.detections
@@ -177,15 +183,18 @@ class MainActivity : ComponentActivity() {
177183
}
178184
}
179185
overlay.setDetections(detections)
180-
setStatus(String.format(
181-
Locale.US,
182-
"Detected %d regions (%d faces, %d plates) in %d ms. Tap/drag boxes; delete false positives.",
183-
detections.size,
184-
detections.count { it.type == DetectionType.FACE },
185-
detections.count { it.type == DetectionType.LICENSE_PLATE },
186-
result.second
187-
))
186+
setStatus(
187+
String.format(
188+
Locale.US,
189+
"Detected %d regions (%d faces, %d plates) in %d ms. Tap/drag boxes; delete false positives.",
190+
detections.size,
191+
detections.count { it.type == DetectionType.FACE },
192+
detections.count { it.type == DetectionType.LICENSE_PLATE },
193+
result.second
194+
)
195+
)
188196
}.onFailure { error ->
197+
progressDialog.dismiss()
189198
setStatus("Detection failed: ${diagnosticMessage(error)}")
190199
}
191200
}

‎demo/src/main/res/layout/dialog.xml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
3+
android:layout_width="match_parent"
4+
android:layout_height="wrap_content"
5+
android:orientation="vertical"
6+
android:padding="24dp">
7+
8+
<TextView
9+
android:id="@+id/autodetect_title"
10+
android:layout_width="match_parent"
11+
android:layout_height="wrap_content"
12+
android:text="Auto-blurring faces and license plates."
13+
android:textSize="18sp"
14+
android:textStyle="bold"
15+
android:textColor="?android:attr/textColorPrimary" />
16+
17+
<TextView
18+
android:id="@+id/autodetect_description"
19+
android:layout_width="match_parent"
20+
android:layout_height="wrap_content"
21+
android:layout_marginTop="8dp"
22+
android:text="You can discard false positives using the ❌ button of each blurring area."
23+
android:textSize="14sp"
24+
android:textColor="?android:attr/textColorSecondary" />
25+
26+
<TextView
27+
android:id="@+id/autodetect_percent"
28+
android:layout_width="match_parent"
29+
android:layout_height="wrap_content"
30+
android:layout_marginTop="16dp"
31+
android:gravity="end"
32+
android:text="0%"
33+
android:textSize="13sp"
34+
android:textStyle="bold"
35+
android:textColor="?android:attr/textColorPrimary" />
36+
37+
<ProgressBar
38+
android:id="@+id/autodetect_progress"
39+
style="?android:attr/progressBarStyleHorizontal"
40+
android:layout_width="match_parent"
41+
android:layout_height="wrap_content"
42+
android:layout_marginTop="6dp"
43+
android:max="100"
44+
android:progress="0"
45+
android:indeterminate="false" />
46+
47+
</LinearLayout>

‎library/src/main/java/org/commons/ml/common/Detection.kt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ import android.graphics.RectF
55

66
/** Low-level detector contract used by the library facade. */
77
interface AiDetector : AutoCloseable {
8-
suspend fun detect(bitmap: Bitmap, options: DetectionOptions = DetectionOptions()): DetectionResult
8+
suspend fun detect(
9+
bitmap: Bitmap,
10+
options: DetectionOptions = DetectionOptions(),
11+
progressListener: DetectionProgressListener? = null
12+
): DetectionResult
913
}
1014

1115
/** Result of running all detector capabilities available on the device. */

‎library/src/main/java/org/commons/ml/common/DetectionProgress.kt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package org.commons.ml.common
2+
3+
/**
4+
* Receives detection progress updates as image chunks are processed.
5+
*
6+
*/
7+
fun interface DetectionProgressListener {
8+
/**
9+
* Called after each image chunk finishes ONNX inference.
10+
*
11+
* @param progress normalized value from 0.0 to 1.0 representing
12+
* the progress of total inference work completed
13+
* across all active detection phases.
14+
*/
15+
fun onProgress(progress: Float)
16+
}

‎library/src/main/java/org/commons/ml/vision/CommonsVision.kt‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.graphics.Bitmap
55
import android.util.Log
66
import org.commons.ml.common.AiDetector
77
import org.commons.ml.common.DetectionOptions
8+
import org.commons.ml.common.DetectionProgressListener
89
import org.commons.ml.common.DetectionResult
910
import org.commons.ml.common.DetectionType
1011
import org.commons.ml.runtime.MlRuntimeException
@@ -26,8 +27,9 @@ class CommonsVision(context: Context) : AutoCloseable {
2627

2728
suspend fun detect(
2829
bitmap: Bitmap,
29-
options: DetectionOptions = DetectionOptions()
30-
): DetectionResult = detector.detect(bitmap, options)
30+
options: DetectionOptions = DetectionOptions(),
31+
progressListener: DetectionProgressListener? = null
32+
): DetectionResult = detector.detect(bitmap, options, progressListener)
3133

3234
override fun close() {
3335
detector.close()
@@ -53,16 +55,26 @@ class CommonsVision(context: Context) : AutoCloseable {
5355
plateInitializationError = plateResult.second
5456
}
5557

56-
override suspend fun detect(bitmap: Bitmap, options: DetectionOptions): DetectionResult {
58+
override suspend fun detect(
59+
bitmap: Bitmap,
60+
options: DetectionOptions,
61+
progressListener: DetectionProgressListener?
62+
): DetectionResult {
5763
checkOpen()
64+
65+
// Face detection takes the first half (0.0 to 0.5) split between any number of chunks.
66+
val faceListener = progressListener?.let { listener ->
67+
DetectionProgressListener { raw -> listener.onProgress(raw * 0.5f) }
68+
}
69+
5870
val faces = if (face == null) {
5971
faceInitializationError?.let {
6072
Log.e(TAG, "Face detector unavailable (${it.code}).", it)
6173
}
6274
emptyList()
6375
} else {
6476
try {
65-
when (val faceResult = face.detect(bitmap, options)) {
77+
when (val faceResult = face.detect(bitmap, options, faceListener)) {
6678
is DetectionResult.Success -> faceResult.detections
6779
is DetectionResult.Partial -> faceResult.detections
6880
is DetectionResult.Unavailable -> return faceResult
@@ -73,14 +85,19 @@ class CommonsVision(context: Context) : AutoCloseable {
7385
}
7486
}
7587

88+
// Plate detection takes the second half (0.5 to 1.0) split between any number of chunks.
89+
val plateListener = progressListener?.let { listener ->
90+
DetectionProgressListener { raw -> listener.onProgress(0.5f + raw * 0.5f) }
91+
}
92+
7693
val plates = if (plate == null) {
7794
plateInitializationError?.let {
7895
Log.e(TAG, "License-plate detector unavailable (${it.code}).", it)
7996
}
8097
emptyList()
8198
} else {
8299
try {
83-
when (val result = plate.detect(bitmap, options)) {
100+
when (val result = plate.detect(bitmap, options, plateListener)) {
84101
is DetectionResult.Success -> result.detections
85102
is DetectionResult.Partial -> result.detections
86103
is DetectionResult.Unavailable -> emptyList()
@@ -91,6 +108,9 @@ class CommonsVision(context: Context) : AutoCloseable {
91108
}
92109
}
93110

111+
// Completion
112+
progressListener?.onProgress(1.0f)
113+
94114
return if (plate == null) {
95115
DetectionResult.Partial(faces + plates, listOf(DetectionType.LICENSE_PLATE))
96116
} else {

‎library/src/main/java/org/commons/ml/vision/OnnxYuNetDetector.kt‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.graphics.RectF
55
import android.util.Log
66
import org.commons.ml.common.Detection
77
import org.commons.ml.common.DetectionOptions
8+
import org.commons.ml.common.DetectionProgressListener
89
import org.commons.ml.common.DetectionResult
910
import org.commons.ml.runtime.ModelInput
1011
import org.commons.ml.runtime.MlRuntimeException
@@ -57,14 +58,20 @@ class OnnxYuNetDetector internal constructor(
5758
}
5859

5960
/** Detects faces or plates and maps model coordinates back to the source bitmap. */
60-
override suspend fun detect(source: Bitmap, options: DetectionOptions): DetectionResult {
61+
override suspend fun detect(
62+
source: Bitmap,
63+
options: DetectionOptions,
64+
progressListener: DetectionProgressListener?
65+
): DetectionResult {
6166
checkOpen()
6267
val threshold = options.confidenceThreshold
63-
val regions =
64-
chunkRegions(source)
65-
val detections = regions.flatMap { region ->
68+
val regions = chunkRegions(source)
69+
val totalChunks = regions.size
70+
val allDetections = mutableListOf<Detection>()
71+
72+
regions.forEachIndexed { index, region ->
6673
val crop = Bitmap.createBitmap(source, region.left, region.top, region.width, region.height)
67-
try {
74+
val chunkDetections = try {
6875
detectRegion(crop, threshold).map { detection ->
6976
detection.copy(
7077
bounds = RectF(detection.bounds).apply {
@@ -77,8 +84,14 @@ class OnnxYuNetDetector internal constructor(
7784
// Never recycle the caller-owned source image.
7885
if (crop !== source) crop.recycle()
7986
}
87+
allDetections += chunkDetections
88+
// Calculated from (0.0 to 1.0)
89+
// Ex: 1st chunk: 1/4(assume) = 0.25 returned to consumer
90+
// which then converts to 0.25*0.50 = 0.125 (12.5% in UI) for individual face or plate progress.
91+
progressListener?.onProgress((index + 1).toFloat() / totalChunks)
8092
}
81-
val result = nonMaximumSuppression(detections).take(options.maximumResults)
93+
94+
val result = nonMaximumSuppression(allDetections).take(options.maximumResults)
8295
return DetectionResult.Success(result)
8396
}
8497

0 commit comments

Comments
 (0)