package expo.modules.facedetection import android.Manifest import android.content.Context import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Canvas import android.graphics.Color import android.graphics.Matrix import android.graphics.Paint import android.graphics.RectF import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.os.Handler import android.os.HandlerThread import android.util.Size import android.view.Surface import android.view.SurfaceHolder import android.view.SurfaceView import android.view.View import android.widget.FrameLayout import android.widget.LinearLayout import androidx.core.content.ContextCompat import expo.modules.kotlin.AppContext import expo.modules.kotlin.viewevent.EventDispatcher import expo.modules.kotlin.views.ExpoView import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean import android.graphics.ImageFormat import android.media.ImageReader import java.nio.ByteBuffer import android.graphics.BitmapFactory import android.graphics.Rect import android.media.Image import android.renderscript.Allocation import android.renderscript.Element import android.renderscript.RenderScript import android.renderscript.ScriptIntrinsicYuvToRGB import android.renderscript.Type class ExpoFaceDetectionView(context: Context, appContext: AppContext) : ExpoView(context, appContext) { private val onMatchResult by EventDispatcher() private val onFaceDetected by EventDispatcher() private val onError by EventDispatcher() private val onEnrollmentCapture by EventDispatcher() private val onEnrollmentComplete by EventDispatcher() private val onEnrollmentStatus by EventDispatcher() private val containerView: FrameLayout private val surfaceView: SurfaceView private val overlayView: FaceOverlayView private var cameraDevice: CameraDevice? = null private var captureSession: CameraCaptureSession? = null private var imageReader: ImageReader? = null private var backgroundHandler: Handler? = null private var backgroundThread: HandlerThread? = null private val manager: FaceDetectionManager by lazy { FaceDetectionManager.getInstance(context) } // Mode: "matching" or "enrollment" private var mode = "matching" // Matching mode props private var matchingEnabled = false private var livenessEnabled = false private var targetEmbedding: FloatArray? = null private var matchThreshold = 1.1f private var cameraFacing = "front" // Enrollment mode state private val enrollmentEmbeddings = mutableListOf() private var enrollmentPhotoIndex = 0 private val enrollmentLabels = listOf("front", "left", "right") private val enrollmentInstructions = listOf( "Look straight at the camera", "Turn your head slightly to the left", "Turn your head slightly to the right" ) private var captureRequested = false private var enrollmentStartTime = 0L // Last frame state for enrollment status updates private var lastFaceDetected = false private var lastIsLive = false private var lastLivenessScore = 0f private var lastFaceBox: Map? = null private var previewSize: Size? = null private var sensorOrientation = 0 private var viewWidth = 0 private var viewHeight = 0 private val isProcessing = AtomicBoolean(false) private val executor = Executors.newSingleThreadExecutor() private var isSettingUpPreview = false // Flag to prevent re-opening camera during preview setup private var pendingCaptureSession = false // Flag to create capture session after surface is ready init { // Create a FrameLayout container to stack SurfaceView and OverlayView containerView = FrameLayout(context) surfaceView = SurfaceView(context) overlayView = FaceOverlayView(context) // Add views to the FrameLayout container (stacked on top of each other) containerView.addView(surfaceView, FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)) containerView.addView(overlayView, FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)) // Add the container to this LinearLayout (ExpoView) addView(containerView, LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.MATCH_PARENT)) surfaceView.holder.addCallback(object : SurfaceHolder.Callback { override fun surfaceCreated(holder: SurfaceHolder) { android.util.Log.d("SURFACE_CALLBACK", "surfaceCreated") startBackgroundThread() } override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { android.util.Log.d("SURFACE_CALLBACK", "surfaceChanged: ${width}x${height}, isSettingUpPreview=$isSettingUpPreview, pendingCaptureSession=$pendingCaptureSession") viewWidth = width viewHeight = height if (pendingCaptureSession) { // Surface was recreated after setFixedSize, now create the capture session android.util.Log.d("SURFACE_CALLBACK", "Surface ready after setFixedSize, creating capture session") pendingCaptureSession = false isSettingUpPreview = false createCaptureSessionNow() } else if (!isSettingUpPreview) { // Normal surface change, open camera openCamera() } else { android.util.Log.d("SURFACE_CALLBACK", "Skipping openCamera - already setting up preview") } } override fun surfaceDestroyed(holder: SurfaceHolder) { android.util.Log.d("SURFACE_CALLBACK", "surfaceDestroyed") closeCamera() stopBackgroundThread() } }) } fun setMode(newMode: String) { if (mode != newMode) { mode = newMode if (mode == "enrollment") { resetEnrollmentState() } } } fun setMatchingEnabled(enabled: Boolean) { matchingEnabled = enabled } fun setLivenessEnabled(enabled: Boolean) { livenessEnabled = enabled } fun setTargetEmbedding(embedding: FloatArray) { targetEmbedding = embedding manager.setTargetEmbedding(embedding) } fun setMatchThreshold(threshold: Float) { matchThreshold = threshold manager.setMatchThreshold(threshold) } fun setCameraFacing(facing: String) { if (cameraFacing != facing) { cameraFacing = facing if (cameraDevice != null) { closeCamera() openCamera() } } } fun setCapturePhoto(capture: Boolean) { if (capture && mode == "enrollment" && !captureRequested) { captureRequested = true android.util.Log.d("ENROLLMENT_DEBUG", "Capture requested for photo ${enrollmentPhotoIndex + 1}") } } fun setResetEnrollment(reset: Boolean) { if (reset && mode == "enrollment") { resetEnrollmentState() android.util.Log.d("ENROLLMENT_DEBUG", "Enrollment reset") } } private fun resetEnrollmentState() { enrollmentEmbeddings.clear() enrollmentPhotoIndex = 0 captureRequested = false enrollmentStartTime = System.currentTimeMillis() lastFaceDetected = false lastIsLive = false lastLivenessScore = 0f lastFaceBox = null // Send initial status post { sendEnrollmentStatus() } } private fun startBackgroundThread() { backgroundThread = HandlerThread("CameraBackground").also { it.start() } backgroundHandler = Handler(backgroundThread!!.looper) } private fun stopBackgroundThread() { backgroundThread?.quitSafely() try { backgroundThread?.join() backgroundThread = null backgroundHandler = null } catch (e: InterruptedException) { e.printStackTrace() } } private fun openCamera() { val TAG = "CAMERA_INIT" android.util.Log.d(TAG, "=== openCamera() START ===") android.util.Log.d(TAG, "viewWidth=$viewWidth, viewHeight=$viewHeight, cameraFacing=$cameraFacing") if (viewWidth == 0 || viewHeight == 0) { android.util.Log.w(TAG, "View dimensions not ready, skipping camera open") return } if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { android.util.Log.e(TAG, "Camera permission not granted") onError(mapOf("error" to "Camera permission not granted")) return } // Close any existing camera before opening a new one android.util.Log.d(TAG, "Closing existing camera if any...") closeCamera() val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager try { val cameraId = getCameraId(cameraManager) android.util.Log.d(TAG, "Selected cameraId=$cameraId") if (cameraId == null) { android.util.Log.e(TAG, "No suitable camera found") onError(mapOf("error" to "No suitable camera found")) return } val characteristics = cameraManager.getCameraCharacteristics(cameraId) val map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) // Get sensor orientation sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 android.util.Log.d(TAG, "sensorOrientation=$sensorOrientation") // Get available sizes and choose optimal one based on view dimensions val outputSizes = map?.getOutputSizes(SurfaceHolder::class.java) ?: arrayOf() android.util.Log.d(TAG, "Available output sizes: ${outputSizes.map { "${it.width}x${it.height}" }}") previewSize = chooseOptimalSize(outputSizes, viewWidth, viewHeight) android.util.Log.d(TAG, "Selected previewSize=${previewSize?.width}x${previewSize?.height}") android.util.Log.d(TAG, "Creating ImageReader...") imageReader = ImageReader.newInstance( previewSize!!.width, previewSize!!.height, ImageFormat.YUV_420_888, 2 ).apply { setOnImageAvailableListener({ reader -> val image = reader.acquireLatestImage() if (image != null && !isProcessing.get()) { // Don't close image here - processImage will close it after processing processImage(image) } else { // Only close if we're not processing this frame image?.close() } }, backgroundHandler) } android.util.Log.d(TAG, "ImageReader created successfully") android.util.Log.d(TAG, "Opening camera...") cameraManager.openCamera(cameraId, object : CameraDevice.StateCallback() { override fun onOpened(camera: CameraDevice) { android.util.Log.d(TAG, "Camera onOpened callback - camera opened successfully") cameraDevice = camera android.util.Log.d(TAG, "Calling createCameraPreview()...") createCameraPreview() } override fun onDisconnected(camera: CameraDevice) { android.util.Log.w(TAG, "Camera onDisconnected callback") camera.close() cameraDevice = null } override fun onError(camera: CameraDevice, error: Int) { android.util.Log.e(TAG, "Camera onError callback - error code: $error") camera.close() cameraDevice = null this@ExpoFaceDetectionView.onError(mapOf("error" to "Camera error: $error")) } }, backgroundHandler) } catch (e: Exception) { android.util.Log.e(TAG, "Failed to open camera", e) onError(mapOf("error" to "Failed to open camera: ${e.message ?: e.javaClass.simpleName}")) } } private fun getCameraId(cameraManager: CameraManager): String? { val facing = if (cameraFacing == "front") { CameraCharacteristics.LENS_FACING_FRONT } else { CameraCharacteristics.LENS_FACING_BACK } for (cameraId in cameraManager.cameraIdList) { val characteristics = cameraManager.getCameraCharacteristics(cameraId) if (characteristics.get(CameraCharacteristics.LENS_FACING) == facing) { return cameraId } } return cameraManager.cameraIdList.firstOrNull() } private fun chooseOptimalSize(choices: Array, viewW: Int, viewH: Int): Size { // For portrait mode, we need to swap width/height for aspect ratio comparison // Camera sensor is typically landscape, view is typically portrait val isPortrait = viewH > viewW val targetRatio = if (isPortrait) { viewH.toDouble() / viewW.toDouble() } else { viewW.toDouble() / viewH.toDouble() } val aspectTolerance = 0.1 val maxPreviewWidth = 1920 val maxPreviewHeight = 1080 // Filter to reasonable sizes val validSizes = choices.filter { it.width <= maxPreviewWidth && it.height <= maxPreviewHeight } // First try to find size with matching aspect ratio val matchingRatioSizes = validSizes.filter { size -> val ratio = size.width.toDouble() / size.height.toDouble() Math.abs(ratio - targetRatio) <= aspectTolerance } // Choose the largest size that matches aspect ratio (up to a reasonable limit) // But not too large to avoid performance issues return if (matchingRatioSizes.isNotEmpty()) { matchingRatioSizes .filter { it.width <= 1280 && it.height <= 960 } .maxByOrNull { it.width * it.height } ?: matchingRatioSizes.minByOrNull { it.width * it.height } ?: validSizes.first() } else { // Fallback: choose a size closest to 640x480 aspect ratio (4:3) validSizes .filter { it.width <= 1280 && it.height <= 960 } .minByOrNull { val ratio = it.width.toDouble() / it.height.toDouble() Math.abs(ratio - 4.0/3.0) + Math.abs(it.width - 640) / 1000.0 } ?: choices.first() } } private fun createCameraPreview() { val TAG = "CAMERA_PREVIEW" android.util.Log.d(TAG, "=== createCameraPreview() START ===") val previewSz = previewSize if (previewSz == null) { android.util.Log.e(TAG, "previewSize is null, cannot create preview") return } android.util.Log.d(TAG, "previewSize=${previewSz.width}x${previewSz.height}") // Mark that we're setting up preview to prevent surfaceChanged from re-opening camera isSettingUpPreview = true // Configure surface layout to maintain aspect ratio // Camera is landscape, view is portrait, so we need to swap for ratio calc val previewAspect = previewSz.width.toFloat() / previewSz.height.toFloat() val viewAspect = viewWidth.toFloat() / viewHeight.toFloat() // For portrait view with landscape camera val rotatedPreviewAspect = previewSz.height.toFloat() / previewSz.width.toFloat() android.util.Log.d(TAG, "previewAspect=$previewAspect, viewAspect=$viewAspect, rotatedPreviewAspect=$rotatedPreviewAspect") // All UI operations must be on main thread post { android.util.Log.d(TAG, "=== Inside post{} on main thread ===") try { // Step 1: Update layout params first (doesn't trigger surface recreation) android.util.Log.d(TAG, "Step 1: Updating layout params...") val params = surfaceView.layoutParams as FrameLayout.LayoutParams // Scale to fill view while maintaining aspect ratio (center-crop style) if (viewAspect > rotatedPreviewAspect) { params.width = viewWidth params.height = (viewWidth / rotatedPreviewAspect).toInt() } else { params.height = viewHeight params.width = (viewHeight * rotatedPreviewAspect).toInt() } params.gravity = android.view.Gravity.CENTER surfaceView.layoutParams = params // Also update overlay to match val overlayParams = overlayView.layoutParams as FrameLayout.LayoutParams overlayParams.width = params.width overlayParams.height = params.height overlayParams.gravity = android.view.Gravity.CENTER overlayView.layoutParams = overlayParams android.util.Log.d(TAG, "Step 1: Done - layout params: ${params.width}x${params.height}") android.util.Log.d(TAG, "=== PREVIEW === view: ${viewWidth}x${viewHeight}, preview: ${previewSz.width}x${previewSz.height}, surface: ${params.width}x${params.height}") // Step 2: Check if we need to change the surface buffer size // setFixedSize may trigger surfaceChanged, so we need to handle it val currentSurface = surfaceView.holder.surface android.util.Log.d(TAG, "Step 2: Current surface isValid=${currentSurface.isValid}") // Set flag to create capture session when surface is ready pendingCaptureSession = true android.util.Log.d(TAG, "Step 3: Setting surface holder fixed size (may trigger surfaceChanged)...") surfaceView.holder.setFixedSize(previewSz.width, previewSz.height) android.util.Log.d(TAG, "Step 3: Done - setFixedSize(${previewSz.width}, ${previewSz.height})") // On some devices, setFixedSize doesn't trigger surfaceChanged // So we post a delayed check to create the capture session if it wasn't created yet postDelayed({ if (pendingCaptureSession && cameraDevice != null && isAttachedToWindow) { android.util.Log.d(TAG, "Step 4: Delayed check - surface didn't change, creating capture session now") pendingCaptureSession = false isSettingUpPreview = false createCaptureSessionNow() } else if (pendingCaptureSession) { android.util.Log.d(TAG, "Step 4: Delayed check - skipping (camera closed or view detached)") pendingCaptureSession = false isSettingUpPreview = false } }, 100) } catch (e: Exception) { isSettingUpPreview = false pendingCaptureSession = false val errorMsg = e.message ?: e.javaClass.simpleName android.util.Log.e(TAG, "Failed to create preview: $errorMsg", e) onError(mapOf("error" to "Failed to create preview: $errorMsg")) } } android.util.Log.d(TAG, "=== createCameraPreview() END (post scheduled) ===") } private fun createCaptureSessionNow() { val TAG = "CAMERA_SESSION" android.util.Log.d(TAG, "=== createCaptureSessionNow() START ===") try { val surface = surfaceView.holder.surface android.util.Log.d(TAG, "Surface isValid=${surface.isValid}") if (!surface.isValid) { android.util.Log.e(TAG, "Surface is not valid, cannot create capture session") onError(mapOf("error" to "Surface is not valid for camera preview")) return } if (cameraDevice == null) { android.util.Log.e(TAG, "cameraDevice is null, cannot create capture session") onError(mapOf("error" to "Camera device not available")) return } if (imageReader == null) { android.util.Log.e(TAG, "imageReader is null, cannot create capture session") onError(mapOf("error" to "Image reader not available")) return } android.util.Log.d(TAG, "Creating capture request...") val previewRequestBuilder = cameraDevice!!.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) previewRequestBuilder.addTarget(surface) previewRequestBuilder.addTarget(imageReader!!.surface) android.util.Log.d(TAG, "Creating capture session...") cameraDevice!!.createCaptureSession( listOf(surface, imageReader!!.surface), object : CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { android.util.Log.d(TAG, "CaptureSession onConfigured") if (cameraDevice == null) { android.util.Log.w(TAG, "cameraDevice became null, closing session") session.close() return } captureSession = session try { previewRequestBuilder.set( CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE ) session.setRepeatingRequest(previewRequestBuilder.build(), null, backgroundHandler) android.util.Log.d(TAG, "Camera preview started successfully!") } catch (e: Exception) { android.util.Log.e(TAG, "Failed to start repeating request", e) } } override fun onConfigureFailed(session: CameraCaptureSession) { android.util.Log.e(TAG, "CaptureSession onConfigureFailed!") this@ExpoFaceDetectionView.onError(mapOf("error" to "Failed to configure camera session")) } }, backgroundHandler ) } catch (e: Exception) { val errorMsg = e.message ?: e.javaClass.simpleName android.util.Log.e(TAG, "Failed to create capture session: $errorMsg", e) onError(mapOf("error" to "Failed to create capture session: $errorMsg")) } } private var frameCount = 0 private fun processImage(image: Image) { val TAG = "FRAME_PROCESS" frameCount++ // Check if we need to process this frame val shouldProcess = when (mode) { "enrollment" -> true // Always process in enrollment mode for status updates "matching" -> matchingEnabled || livenessEnabled else -> false } if (!shouldProcess) { image.close() return } // Log every 30 frames to avoid spam if (frameCount % 30 == 1) { android.util.Log.d(TAG, "Processing frame #$frameCount, mode=$mode, image=${image.width}x${image.height}") } isProcessing.set(true) executor.execute { try { // Convert image to bitmap first, then close the image val imgWidth = image.width val imgHeight = image.height if (frameCount % 30 == 1) { android.util.Log.d(TAG, "Converting YUV to bitmap...") } val bitmap = imageToBitmap(image) image.close() // Close image immediately after conversion if (frameCount % 30 == 1) { android.util.Log.d(TAG, "Bitmap created: ${bitmap.width}x${bitmap.height}, processing mode=$mode") } when (mode) { "enrollment" -> processEnrollmentFrame(bitmap) "matching" -> processMatchingFrame(bitmap) } bitmap.recycle() } catch (e: Exception) { try { image.close() } catch (_: Exception) {} val errorMsg = e.message ?: "${e.javaClass.simpleName} at ${e.stackTrace.firstOrNull()?.toString() ?: "unknown"}" android.util.Log.e(TAG, "Processing error at frame #$frameCount: $errorMsg", e) android.util.Log.e(TAG, "Stack trace: ${e.stackTraceToString()}") post { onError(mapOf("error" to "Processing error: $errorMsg")) } } finally { isProcessing.set(false) } } } private fun processEnrollmentFrame(bitmap: Bitmap) { val TAG = "ENROLLMENT_DEBUG" // Check liveness and face detection on current frame val livenessResult = manager.checkLiveness(bitmap) // Update last frame state lastFaceDetected = livenessResult.faceDetected lastIsLive = livenessResult.isLive lastLivenessScore = livenessResult.livenessScore lastFaceBox = livenessResult.faceBox // Update overlay post { if (livenessResult.faceDetected) { overlayView.setFaceBox(livenessResult.faceBox, bitmap.width, bitmap.height, false) } else { overlayView.clearFaceBox() } } // Check if capture was requested if (captureRequested) { captureRequested = false // Reset immediately android.util.Log.d(TAG, "Processing capture for photo ${enrollmentPhotoIndex + 1}") if (!livenessResult.faceDetected) { // No face detected, send error post { onEnrollmentCapture(mapOf( "photoIndex" to enrollmentPhotoIndex, "photoLabel" to enrollmentLabels[enrollmentPhotoIndex], "totalPhotos" to 3, "success" to false, "faceDetected" to false, "errorMessage" to "No face detected. Please position your face in the frame." )) } return } // Extract embedding from this frame val embeddingResult = manager.extractEmbedding(bitmap) if (!embeddingResult.success || embeddingResult.embedding == null) { post { onEnrollmentCapture(mapOf( "photoIndex" to enrollmentPhotoIndex, "photoLabel" to enrollmentLabels[enrollmentPhotoIndex], "totalPhotos" to 3, "success" to false, "faceDetected" to true, "errorMessage" to (embeddingResult.errorMessage ?: "Failed to extract face embedding") )) } return } // Store the embedding val embedding = embeddingResult.embedding.map { it.toFloat() }.toFloatArray() enrollmentEmbeddings.add(embedding) // Capture current index before incrementing (for async callback) val capturedPhotoIndex = enrollmentPhotoIndex val capturedPhotoLabel = enrollmentLabels[capturedPhotoIndex] android.util.Log.d(TAG, "Photo ${capturedPhotoIndex + 1} captured successfully. Embedding norm: ${kotlin.math.sqrt(embedding.map { it * it }.sum())}") // Send success callback post { onEnrollmentCapture(mutableMapOf( "photoIndex" to capturedPhotoIndex, "photoLabel" to capturedPhotoLabel, "totalPhotos" to 3, "success" to true, "faceDetected" to true, "isLive" to livenessResult.isLive, "livenessScore" to livenessResult.livenessScore ).apply { livenessResult.faceBox?.let { put("faceBox", it) } }) } // Move to next photo enrollmentPhotoIndex++ // Check if all photos captured if (enrollmentPhotoIndex >= 3) { finalizeEnrollment() } else { // Send updated status for next photo post { sendEnrollmentStatus() } } } else { // Just send status update (not capturing) post { sendEnrollmentStatus() } } } private fun finalizeEnrollment() { val TAG = "ENROLLMENT_DEBUG" android.util.Log.d(TAG, "Finalizing enrollment with ${enrollmentEmbeddings.size} embeddings") if (enrollmentEmbeddings.size != 3) { post { onEnrollmentComplete(mapOf( "success" to false, "photoCount" to enrollmentEmbeddings.size, "processingTimeMs" to (System.currentTimeMillis() - enrollmentStartTime), "errorMessage" to "Expected 3 embeddings, got ${enrollmentEmbeddings.size}" )) } return } // Average the embeddings val avgEmbedding = FloatArray(192) for (i in 0 until 192) { avgEmbedding[i] = (enrollmentEmbeddings[0][i] + enrollmentEmbeddings[1][i] + enrollmentEmbeddings[2][i]) / 3f } // Re-normalize val norm = kotlin.math.sqrt(avgEmbedding.map { it * it }.sum()) android.util.Log.d(TAG, "Averaged embedding norm before normalization: $norm") if (norm > 1e-10) { for (i in 0 until 192) { avgEmbedding[i] = avgEmbedding[i] / norm.toFloat() } } val finalNorm = kotlin.math.sqrt(avgEmbedding.map { it * it }.sum()) android.util.Log.d(TAG, "Final embedding norm: $finalNorm") // Convert to List for JS val embeddingList = avgEmbedding.map { it.toDouble() } post { onEnrollmentComplete(mapOf( "success" to true, "embedding" to embeddingList, "photoCount" to 3, "processingTimeMs" to (System.currentTimeMillis() - enrollmentStartTime) )) } // Reset for potential re-enrollment resetEnrollmentState() } private fun sendEnrollmentStatus() { if (mode != "enrollment") return val readyToCapture = lastFaceDetected && (enrollmentPhotoIndex < 3) onEnrollmentStatus(mutableMapOf( "currentPhotoIndex" to enrollmentPhotoIndex, "photoLabel" to enrollmentLabels.getOrElse(enrollmentPhotoIndex) { "complete" }, "instruction" to enrollmentInstructions.getOrElse(enrollmentPhotoIndex) { "Enrollment complete" }, "photosRemaining" to (3 - enrollmentPhotoIndex), "readyToCapture" to readyToCapture, "faceDetected" to lastFaceDetected, "isLive" to lastIsLive, "livenessScore" to lastLivenessScore ).apply { lastFaceBox?.let { put("faceBox", it) } }) } private fun processMatchingFrame(bitmap: Bitmap) { val TAG = "LIVE_PROCESS_DEBUG" android.util.Log.d(TAG, "=== LIVE FRAME START ===") android.util.Log.d(TAG, "Camera: $cameraFacing, Bitmap: ${bitmap.width}x${bitmap.height}") val mirroredBitmap = if (cameraFacing == "front") { mirrorBitmap(bitmap) } else { bitmap } if (livenessEnabled) { val livenessResult = manager.checkLiveness(mirroredBitmap) if (livenessResult.faceDetected) { post { overlayView.setFaceBox(livenessResult.faceBox, mirroredBitmap.width, mirroredBitmap.height) onFaceDetected(mutableMapOf( "faceDetected" to true, "isLive" to livenessResult.isLive, "livenessScore" to livenessResult.livenessScore, "sharpness" to livenessResult.sharpness, "confidence" to livenessResult.confidence ).apply { livenessResult.faceBox?.let { put("faceBox", it) } }) } } else { post { overlayView.clearFaceBox() } } } if (matchingEnabled && targetEmbedding != null) { // Use the same bitmap orientation as enrollment (non-mirrored) // Since enrollment now uses the same native camera, both are consistent val matchBitmap = bitmap android.util.Log.d(TAG, "--- MATCHING WITH BITMAP ${matchBitmap.width}x${matchBitmap.height} ---") val matchResult = manager.processFrame(matchBitmap) android.util.Log.d(TAG, "=== MATCH RESULT === detected: ${matchResult.faceDetected}, distance: ${matchResult.distance}, threshold: $matchThreshold, isMatch: ${matchResult.isMatch}") post { if (matchResult.faceDetected) { val displayFaceBox = if (cameraFacing == "front" && matchResult.faceBox != null) { val box = matchResult.faceBox!! val left = box["left"] val right = box["right"] val top = box["top"] val bottom = box["bottom"] if (left != null && right != null && top != null && bottom != null) { val leftVal = (left as Number).toInt() val rightVal = (right as Number).toInt() mapOf( "left" to (bitmap.width - rightVal), "top" to (top as Number).toInt(), "right" to (bitmap.width - leftVal), "bottom" to (bottom as Number).toInt() ) } else { matchResult.faceBox } } else { matchResult.faceBox } overlayView.setFaceBox(displayFaceBox, bitmap.width, bitmap.height, matchResult.isMatch) } else { overlayView.clearFaceBox() } onMatchResult(mutableMapOf( "faceDetected" to matchResult.faceDetected, "isMatch" to matchResult.isMatch, "confidence" to matchResult.confidence, "distance" to matchResult.distance, "processingTimeMs" to matchResult.processingTimeMs ).apply { matchResult.faceBox?.let { put("faceBox", it) } matchResult.errorMessage?.let { put("errorMessage", it) } }) } } if (mirroredBitmap != bitmap) { mirroredBitmap.recycle() } } private fun imageToBitmap(image: Image): Bitmap { val TAG = "CAMERA_DEBUG" val width = image.width val height = image.height android.util.Log.d(TAG, "=== YUV TO BITMAP START ===") android.util.Log.d(TAG, "Image: ${width}x${height}, format: ${image.format}") val yPlane = image.planes[0] val uPlane = image.planes[1] val vPlane = image.planes[2] // Create duplicate buffers to avoid modifying original positions val yBuffer = yPlane.buffer.duplicate() val uBuffer = uPlane.buffer.duplicate() val vBuffer = vPlane.buffer.duplicate() // Rewind buffers to start from beginning yBuffer.rewind() uBuffer.rewind() vBuffer.rewind() val yRowStride = yPlane.rowStride val uvRowStride = uPlane.rowStride val uvPixelStride = uPlane.pixelStride android.util.Log.d(TAG, "Y plane: rowStride=$yRowStride, capacity=${yBuffer.capacity()}, remaining=${yBuffer.remaining()}") android.util.Log.d(TAG, "U plane: rowStride=$uvRowStride, pixelStride=$uvPixelStride, capacity=${uBuffer.capacity()}") android.util.Log.d(TAG, "V plane: capacity=${vBuffer.capacity()}") // Create NV21 byte array with proper stride handling val nv21 = ByteArray(width * height * 3 / 2) // Copy Y plane row by row (handles padding) var pos = 0 for (row in 0 until height) { val rowStart = row * yRowStride // Safety check to avoid buffer overflow if (rowStart + width <= yBuffer.capacity()) { yBuffer.position(rowStart) yBuffer.get(nv21, pos, width) } else { // Fallback: read what we can from remaining yBuffer.position(rowStart.coerceAtMost(yBuffer.capacity() - 1)) val bytesToRead = minOf(width, yBuffer.remaining()) if (bytesToRead > 0) { yBuffer.get(nv21, pos, bytesToRead) } } pos += width } // Copy UV planes (interleaved as VU for NV21) val uvHeight = height / 2 val uvWidth = width / 2 // Manual interleaving with bounds checking for (row in 0 until uvHeight) { for (col in 0 until uvWidth) { val uvIndex = row * uvRowStride + col * uvPixelStride // Safety check for buffer bounds if (uvIndex < vBuffer.capacity() && uvIndex < uBuffer.capacity()) { nv21[pos++] = vBuffer.get(uvIndex) nv21[pos++] = uBuffer.get(uvIndex) } else { // Fallback to zero (black) if out of bounds nv21[pos++] = 128.toByte() // V = 128 (neutral) nv21[pos++] = 128.toByte() // U = 128 (neutral) } } } val yuvImage = android.graphics.YuvImage(nv21, ImageFormat.NV21, width, height, null) val out = java.io.ByteArrayOutputStream() yuvImage.compressToJpeg(Rect(0, 0, width, height), 90, out) val imageBytes = out.toByteArray() val bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size) android.util.Log.d(TAG, "JPEG size: ${imageBytes.size} bytes") android.util.Log.d(TAG, "Bitmap before rotation: ${bitmap.width}x${bitmap.height}, config: ${bitmap.config}") // Sample pixel for color verification val centerPixel = bitmap.getPixel(bitmap.width / 2, bitmap.height / 2) val r = (centerPixel shr 16) and 0xFF val g = (centerPixel shr 8) and 0xFF val b = centerPixel and 0xFF android.util.Log.d(TAG, "Pre-rotation center pixel RGB: ($r, $g, $b)") // Apply rotation based on sensor orientation val rotationDegrees = getRotationCompensation() val result = if (rotationDegrees != 0) { val matrix = Matrix() matrix.postRotate(rotationDegrees.toFloat()) val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) android.util.Log.d(TAG, "Rotated by $rotationDegrees degrees: ${rotated.width}x${rotated.height}") rotated } else { android.util.Log.d(TAG, "No rotation needed") bitmap } android.util.Log.d(TAG, "=== YUV TO BITMAP END ===") return result } private fun getRotationCompensation(): Int { // Get device rotation val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as android.view.WindowManager val deviceRotation = windowManager.defaultDisplay.rotation val deviceDegrees = when (deviceRotation) { Surface.ROTATION_0 -> 0 Surface.ROTATION_90 -> 90 Surface.ROTATION_180 -> 180 Surface.ROTATION_270 -> 270 else -> 0 } // Calculate rotation compensation return if (cameraFacing == "front") { // Front camera rotation compensation val rotation = (sensorOrientation + deviceDegrees) % 360 (360 - rotation) % 360 // Compensate for front camera mirror } else { (sensorOrientation - deviceDegrees + 360) % 360 } } private fun mirrorBitmap(bitmap: Bitmap): Bitmap { val matrix = Matrix() matrix.preScale(-1f, 1f) return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) } private fun closeCamera() { android.util.Log.d("CAMERA_INIT", "closeCamera() called") isSettingUpPreview = false pendingCaptureSession = false captureSession?.close() captureSession = null cameraDevice?.close() cameraDevice = null imageReader?.close() imageReader = null } override fun onDetachedFromWindow() { super.onDetachedFromWindow() closeCamera() stopBackgroundThread() executor.shutdown() } // Face overlay view for drawing bounding boxes private inner class FaceOverlayView(context: Context) : View(context) { private val paint = Paint().apply { style = Paint.Style.STROKE strokeWidth = 4f color = Color.GREEN } private var faceRect: RectF? = null private var isMatch = false private var imageWidth = 0 private var imageHeight = 0 fun setFaceBox(box: Map?, imgWidth: Int, imgHeight: Int, match: Boolean = false) { if (box == null) { clearFaceBox() return } imageWidth = imgWidth imageHeight = imgHeight isMatch = match val left = box["left"] ?: 0 val top = box["top"] ?: 0 val right = box["right"] ?: 0 val bottom = box["bottom"] ?: 0 // Scale to view coordinates val scaleX = width.toFloat() / imageWidth val scaleY = height.toFloat() / imageHeight faceRect = RectF( left * scaleX, top * scaleY, right * scaleX, bottom * scaleY ) invalidate() } fun clearFaceBox() { faceRect = null invalidate() } override fun onDraw(canvas: Canvas) { super.onDraw(canvas) faceRect?.let { rect -> paint.color = if (isMatch) Color.GREEN else Color.YELLOW canvas.drawRect(rect, paint) } } } }