package com.visioncamerafacedetection import android.graphics.Bitmap import android.graphics.Canvas import android.graphics.Matrix import android.graphics.Rect import android.graphics.RectF import android.util.Log import android.view.Surface import com.google.android.gms.tasks.Tasks import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.common.internal.ImageConvertUtils import com.google.mlkit.vision.face.Face import com.google.mlkit.vision.face.FaceContour import com.google.mlkit.vision.face.FaceDetection import com.google.mlkit.vision.face.FaceDetector import com.google.mlkit.vision.face.FaceDetectorOptions import com.google.mlkit.vision.face.FaceLandmark import com.mrousavy.camera.core.FrameInvalidError import com.mrousavy.camera.core.types.Orientation import com.mrousavy.camera.core.types.Position import com.mrousavy.camera.frameprocessors.Frame import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin import com.mrousavy.camera.frameprocessors.VisionCameraProxy import java.nio.ByteBuffer import java.nio.FloatBuffer private const val TAG = "FaceDetector" class VisionCameraFaceDetectionPlugin( proxy: VisionCameraProxy, options: Map? ) : FrameProcessorPlugin() { // detection props private var autoMode = false private var faceDetector: FaceDetector? = null private var runLandmarks = false private var runClassifications = false private var runContours = false private var trackingEnabled = false private var windowWidth = 1.0 private var windowHeight = 1.0 private var cameraFacing: Position = Position.FRONT private val orientationManager = VisionCameraFaceDetectorOrientation(proxy.context) private var enableTensor = false init { // handle auto scaling autoMode = options?.get("autoMode").toString() == "true" // handle enable/disable tensor enableTensor = options?.get("enableTensor").toString() == "true" // initializes faceDetector on creation var performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_FAST var landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_NONE var classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_NONE var contourModeValue = FaceDetectorOptions.CONTOUR_MODE_NONE windowWidth = (options?.get("windowWidth") ?: 1.0) as Double windowHeight = (options?.get("windowHeight") ?: 1.0) as Double if (options?.get("cameraFacing").toString() == "back") { cameraFacing = Position.BACK } if (options?.get("performanceMode").toString() == "accurate") { performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE } if (options?.get("landmarkMode").toString() == "all") { runLandmarks = true landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_ALL } if (options?.get("classificationMode").toString() == "all") { runClassifications = true classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_ALL } if (options?.get("contourMode").toString() == "all") { runContours = true contourModeValue = FaceDetectorOptions.CONTOUR_MODE_ALL } val minFaceSize: Double = (options?.get("minFaceSize") ?: 0.15) as Double val optionsBuilder = FaceDetectorOptions.Builder() .setPerformanceMode(performanceModeValue) .setLandmarkMode(landmarkModeValue) .setContourMode(contourModeValue) .setClassificationMode(classificationModeValue) .setMinFaceSize(minFaceSize.toFloat()) if (options?.get("trackingEnabled").toString() == "true") { trackingEnabled = true optionsBuilder.enableTracking() } faceDetector = FaceDetection.getClient( optionsBuilder.build() ) } private fun processBoundingBox( boundingBox: Rect, sourceWidth: Double, sourceHeight: Double, scaleX: Double, scaleY: Double ): Map { val bounds: MutableMap = HashMap() val width = boundingBox.width().toDouble() * scaleX val height = boundingBox.height().toDouble() * scaleY val x = boundingBox.left.toDouble() val y = boundingBox.top.toDouble() bounds["width"] = width bounds["height"] = height bounds["x"] = x * scaleX bounds["y"] = y * scaleY if(!autoMode) return bounds // using front camera if(cameraFacing == Position.FRONT) { when (orientationManager.orientation) { // device is portrait Surface.ROTATION_0 -> { bounds["x"] = ((-x * scaleX) + sourceWidth * scaleX) - width bounds["y"] = y * scaleY } // device is landscape right Surface.ROTATION_270 -> { bounds["x"] = y * scaleX bounds["y"] = x * scaleY } // device is upside down Surface.ROTATION_180 -> { bounds["x"] = x * scaleX bounds["y"] = ((-y * scaleY) + sourceHeight * scaleY) - height } // device is landscape left Surface.ROTATION_90 -> { bounds["x"] = ((-y * scaleX) + sourceWidth * scaleX) - width bounds["y"] = ((-x * scaleY) + sourceHeight * scaleY) - height } } return bounds } // using back camera when (orientationManager.orientation) { // device is portrait Surface.ROTATION_0 -> { bounds["x"] = x * scaleX bounds["y"] = y * scaleY } // device is landscape right Surface.ROTATION_270 -> { bounds["x"] = y * scaleX bounds["y"] = ((-x * scaleY) + sourceHeight * scaleY) - height } // device is upside down Surface.ROTATION_180 -> { bounds["x"] =((-x * scaleX) + sourceWidth * scaleX) - width bounds["y"] = ((-y * scaleY) + sourceHeight * scaleY) - height } // device is landscape left Surface.ROTATION_90 -> { bounds["x"] = ((-y * scaleX) + sourceWidth * scaleX) - width bounds["y"] = x * scaleY } } return bounds } private fun processLandmarks( face: Face, scaleX: Double, scaleY: Double ): Map { val faceLandmarksTypes = intArrayOf( FaceLandmark.LEFT_CHEEK, FaceLandmark.LEFT_EAR, FaceLandmark.LEFT_EYE, FaceLandmark.MOUTH_BOTTOM, FaceLandmark.MOUTH_LEFT, FaceLandmark.MOUTH_RIGHT, FaceLandmark.NOSE_BASE, FaceLandmark.RIGHT_CHEEK, FaceLandmark.RIGHT_EAR, FaceLandmark.RIGHT_EYE ) val faceLandmarksTypesStrings = arrayOf( "LEFT_CHEEK", "LEFT_EAR", "LEFT_EYE", "MOUTH_BOTTOM", "MOUTH_LEFT", "MOUTH_RIGHT", "NOSE_BASE", "RIGHT_CHEEK", "RIGHT_EAR", "RIGHT_EYE" ) val faceLandmarksTypesMap: MutableMap = HashMap() for (i in faceLandmarksTypesStrings.indices) { val landmark = face.getLandmark(faceLandmarksTypes[i]) val landmarkName = faceLandmarksTypesStrings[i] if (landmark == null) continue val point = landmark.position val currentPointsMap: MutableMap = HashMap() currentPointsMap["x"] = point.x.toDouble() * scaleX currentPointsMap["y"] = point.y.toDouble() * scaleY faceLandmarksTypesMap[landmarkName] = currentPointsMap } return faceLandmarksTypesMap } private fun processFaceContours( face: Face, scaleX: Double, scaleY: Double ): Map { val faceContoursTypes = intArrayOf( FaceContour.FACE, FaceContour.LEFT_CHEEK, FaceContour.LEFT_EYE, FaceContour.LEFT_EYEBROW_BOTTOM, FaceContour.LEFT_EYEBROW_TOP, FaceContour.LOWER_LIP_BOTTOM, FaceContour.LOWER_LIP_TOP, FaceContour.NOSE_BOTTOM, FaceContour.NOSE_BRIDGE, FaceContour.RIGHT_CHEEK, FaceContour.RIGHT_EYE, FaceContour.RIGHT_EYEBROW_BOTTOM, FaceContour.RIGHT_EYEBROW_TOP, FaceContour.UPPER_LIP_BOTTOM, FaceContour.UPPER_LIP_TOP ) val faceContoursTypesStrings = arrayOf( "FACE", "LEFT_CHEEK", "LEFT_EYE", "LEFT_EYEBROW_BOTTOM", "LEFT_EYEBROW_TOP", "LOWER_LIP_BOTTOM", "LOWER_LIP_TOP", "NOSE_BOTTOM", "NOSE_BRIDGE", "RIGHT_CHEEK", "RIGHT_EYE", "RIGHT_EYEBROW_BOTTOM", "RIGHT_EYEBROW_TOP", "UPPER_LIP_BOTTOM", "UPPER_LIP_TOP" ) val faceContoursTypesMap: MutableMap = HashMap() for (i in faceContoursTypesStrings.indices) { val contour = face.getContour(faceContoursTypes[i]) val contourName = faceContoursTypesStrings[i] if (contour == null) continue val points = contour.points val pointsMap: MutableList> = mutableListOf() for (j in points.indices) { val currentPointsMap: MutableMap = HashMap() currentPointsMap["x"] = points[j].x.toDouble() * scaleX currentPointsMap["y"] = points[j].y.toDouble() * scaleY pointsMap.add(currentPointsMap) } faceContoursTypesMap[contourName] = pointsMap } return faceContoursTypesMap } private fun getImageOrientation(): Int { return when (orientationManager.orientation) { // device is portrait Surface.ROTATION_0 -> if(cameraFacing == Position.FRONT) 270 else 90 // device is landscape right Surface.ROTATION_270 -> if(cameraFacing == Position.FRONT) 180 else 180 // device is upside down Surface.ROTATION_180 -> if(cameraFacing == Position.FRONT) 90 else 270 // device is landscape left Surface.ROTATION_90 -> if(cameraFacing == Position.FRONT) 0 else 0 else -> 0 } } override fun callback( frame: Frame, params: Map? ): Any { val result = ArrayList>() try { val image = InputImage.fromMediaImage(frame.image, getImageOrientation()) // we need to invert sizes as frame is always -90deg rotated val width = image.height.toDouble() val height = image.width.toDouble() val scaleX = if(autoMode) windowWidth / width else 1.0 val scaleY = if(autoMode) windowHeight / height else 1.0 val task = faceDetector!!.process(image) val faces = Tasks.await(task) faces.forEach{face -> val map: MutableMap = HashMap() val arrayData: MutableList = ArrayList() if (enableTensor) { val bmpFrameResult = ImageConvertUtils.getInstance().getUpRightBitmap(image) val bmpFaceResult = Bitmap.createBitmap( TF_OD_API_INPUT_SIZE, TF_OD_API_INPUT_SIZE, Bitmap.Config.ARGB_8888 ) val faceBB = RectF(face.boundingBox) val cvFace = Canvas(bmpFaceResult) val sx = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.width() val sy = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.height() val matrix = Matrix() matrix.postTranslate(-faceBB.left, -faceBB.top) matrix.postScale(sx, sy) cvFace.drawBitmap(bmpFrameResult, matrix, null) val input: ByteBuffer = FaceHelper().bitmap2ByteBuffer(bmpFaceResult) val output: FloatBuffer = FloatBuffer.allocate(512) interpreter?.run(input, output) for (i: Float in output.array()) { arrayData.add(i.toDouble()) } map["data"] = arrayData } else { map["data"] = arrayData } if (runLandmarks) { map["landmarks"] = processLandmarks( face, scaleX, scaleY ) } if (runClassifications) { map["leftEyeOpenProbability"] = face.leftEyeOpenProbability?.toDouble() ?: -1 map["rightEyeOpenProbability"] = face.rightEyeOpenProbability?.toDouble() ?: -1 map["smilingProbability"] = face.smilingProbability?.toDouble() ?: -1 } if (runContours) { map["contours"] = processFaceContours( face, scaleX, scaleY ) } if (trackingEnabled) { map["trackingId"] = face.trackingId ?: -1 } map["rollAngle"] = face.headEulerAngleZ.toDouble() map["pitchAngle"] = face.headEulerAngleX.toDouble() map["yawAngle"] = face.headEulerAngleY.toDouble() map["bounds"] = processBoundingBox( face.boundingBox, width, height, scaleX, scaleY ) result.add(map) } } catch (e: Exception) { Log.e(TAG, "Error processing face detection: ", e) } catch (e: FrameInvalidError) { Log.e(TAG, "Frame invalid error: ", e) } return result } }