Skip to content

ADFA-3912 | Fix fuzzy attribute parsing bounds and UI element tag filtering #1275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jatezzz wants to merge 1 commit into
base: stage
Choose a base branch
from
+90 −37
Open

ADFA-3912 | Fix fuzzy attribute parsing bounds and UI element tag filtering #1275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .../src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ object FuzzyAttributeParser {
annotation.length
}

if (current.valueStart > valueEnd) continue

var rawValue = annotation.substring(current.valueStart, valueEnd).trim()

if (rawValue.isNotEmpty()) {
Expand Down
111 changes: 74 additions & 37 deletions ...ml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox
import org.appdevforall.codeonthego.computervision.domain.xml.AndroidXmlGenerator
import org.appdevforall.codeonthego.computervision.utils.MetadataDetector
import org.appdevforall.codeonthego.computervision.utils.getSortedScaledPlaceholders
import org.appdevforall.codeonthego.computervision.utils.buildPlaceholderOverrides
import kotlin.comparisons.compareBy

class YoloToXmlConverter(
Expand All @@ -23,41 +23,31 @@ class YoloToXmlConverter(
targetDpHeight: Int,
wrapInScroll: Boolean = true
): Pair<String, String> {
val uiCandidates = detections
.filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" }
.filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) }
.distinctBy {
if (it.label.startsWith("switch")) {
"${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}"
} else {
"${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}"
}
}
// 1. Filter and prepare base UI candidates
val uiCandidates = extractUiCandidates(detections)

var scaledBoxes = uiCandidates.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) }
// 2. Scale detections to target DP dimensions
val scaledBoxes = scaleDetections(uiCandidates, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight)

val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) }
var texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
// 3. Associate isolated text detections to their respective UI widgets
val associatedBoxes = associateTextToWidgets(scaledBoxes)

scaledBoxes = geometryProcessor.assignTextToParents(parents, texts, scaledBoxes)
texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
scaledBoxes = geometryProcessor.assignNearbyTextToWidgets(scaledBoxes, texts)
// 4. Clean up and finalize the UI elements list
val uiElements = finalizeUiElements(associatedBoxes)

val uiElements = scaledBoxes.filter {
!annotationMatcher.isTag(it.text) && !MetadataDetector.isMetadataDetection(it.label, it.text)
}
val widgetTags = detections.filter { it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text)) }
val canvasTags = widgetTags.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) }
// 5. Extract and scale reference tags (e.g., T-1, B-1) from the canvas
val canvasTags = extractCanvasTags(detections, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight)

// 6. Match margin annotations with the extracted UI elements
val finalAnnotations = annotationMatcher.matchAnnotationsToElements(canvasTags, uiElements, annotations)

// 7. Sort boxes top-to-bottom, left-to-right for sequential XML rendering
val sortedBoxes = uiElements.sortedWith(compareBy({ it.y }, { it.x }))

val selectedImageOverrides = buildSelectedImageOverrides(
uiElements = uiElements,
selectedImagesByPlaceholderId = selectedImagesByPlaceholderId
)
// 8. Prepare local drawable resources overrides for image placeholders
val selectedImageOverrides = uiElements.buildPlaceholderOverrides(selectedImagesByPlaceholderId)

// 9. Generate final XML output
return xmlGenerator.buildXml(
boxes = sortedBoxes,
annotations = finalAnnotations,
Expand All @@ -67,17 +57,64 @@ class YoloToXmlConverter(
)
}

private fun buildSelectedImageOverrides(
uiElements: List<ScaledBox>,
selectedImagesByPlaceholderId: Map<String, String>
): Map<ScaledBox, String> {
val placeholders = uiElements.getSortedScaledPlaceholders()

return placeholders.mapIndexedNotNull { index, box ->
val drawableReference = selectedImagesByPlaceholderId["ph_$index"]
?: return@mapIndexedNotNull null
box to drawableReference
}.toMap()
private fun extractUiCandidates(detections: List<DetectionResult>): List<DetectionResult> {
return detections
.filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" }
.filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) }
.distinctBy {
if (it.label.startsWith("switch")) {
// Deduplicate switches by grouping them within a 50px vertical band
"${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}"
} else {
// Exact coordinate deduplication for other widgets
"${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}"
}
}
}

private fun scaleDetections(
candidates: List<DetectionResult>,
sourceWidth: Int,
sourceHeight: Int,
targetWidth: Int,
targetHeight: Int
): List<ScaledBox> {
return candidates.map {
geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight)
}
}

private fun associateTextToWidgets(scaledBoxes: List<ScaledBox>): List<ScaledBox> {
val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) }
val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }

val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes)

val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts)
}
Comment on lines +87 to +95
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Restrict tag filtering to text boxes here too.

Line 88 still excludes any non-text box whose OCR text looks like a tag. That can drop valid widgets before text association, so the tag-filtering bug is still partially present despite the later fix in finalizeUiElements.

Suggested fix 
 private fun associateTextToWidgets(scaledBoxes: List<ScaledBox>): List<ScaledBox> {
-    val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) }
+    val parents = scaledBoxes.filter { it.label != "text" }
     val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
 
     val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
private fun associateTextToWidgets(scaledBoxes: List<ScaledBox>): List<ScaledBox> {
val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) }
val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes)
val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts)
}
private fun associateTextToWidgets(scaledBoxes: List<ScaledBox>): List<ScaledBox> {
val parents = scaledBoxes.filter { it.label != "text" }
val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes)
val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts)
}
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In
`@cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt`
around lines 87 - 95, In associateTextToWidgets change the parent selection so
you do not drop non-text boxes based on annotationMatcher.isTag: keep parents as
all boxes where it.label != "text" (remove the isTag check), while still
filtering initialTexts with annotationMatcher.isTag so only text boxes get
tag-filtered; then call geometryProcessor.assignTextToParents(parents,
initialTexts, scaledBoxes) and proceed with the existing remainingTexts and
geometryProcessor.assignNearbyTextToWidgets call unchanged (references:
associateTextToWidgets, annotationMatcher.isTag,
geometryProcessor.assignTextToParents,
geometryProcessor.assignNearbyTextToWidgets).


private fun finalizeUiElements(boxes: List<ScaledBox>): List<ScaledBox> {
return boxes.filter {
// Keep the widget if it's not pure text, or if it is text but not recognized as a tag.
(it.label != "text" || !annotationMatcher.isTag(it.text)) &&
!MetadataDetector.isMetadataDetection(it.label, it.text)
}
}

private fun extractCanvasTags(
detections: List<DetectionResult>,
sourceWidth: Int,
sourceHeight: Int,
targetWidth: Int,
targetHeight: Int
): List<ScaledBox> {
val widgetTags = detections.filter {
it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text))
}
return widgetTags.map {
geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight)
}
}

companion object {
Expand Down
14 changes: 14 additions & 0 deletions ...o-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,17 @@ fun List<ScaledBox>.getSortedScaledPlaceholders(): List<ScaledBox> {
return this.filter { it.label == IMAGE_PLACEHOLDER_LABEL }
.sortedWith(compareBy({ it.y }, { it.x }))
}

/**
* Associates ordered image placeholders with their selected drawable references.
* Useful for mapping user-selected gallery images to the physical canvas bounding boxes.
*/
fun List<ScaledBox>.buildPlaceholderOverrides(selectedImagesByPlaceholderId: Map<String, String>): Map<ScaledBox, String> {
val placeholders = this.getSortedScaledPlaceholders()

return placeholders.mapIndexedNotNull { index, box ->
val drawableReference = selectedImagesByPlaceholderId["ph_$index"]
?: return@mapIndexedNotNull null
box to drawableReference
}.toMap()
}
Loading