import { View, Platform, useWindowDimensions, type LayoutChangeEvent, } from 'react-native'; import React, { memo, useRef, useEffect } from 'react'; import { useResizePlugin } from 'vision-camera-resize-plugin'; import { Camera, runAtTargetFps, useFrameProcessor, type Point, } from 'react-native-vision-camera'; import Logger from '../../utils/logger'; import { SelfieError, SelfieErrorCode, type InitialiseSelfieProps, type SelfieScanHandle, } from '../../types/sdkTypes'; import type { TensorflowModel } from '@get-id/react-native-fast-tflite'; import { useCameraSetup } from '../../hooks'; import CameraStyles from '../../styles/CameraStyles'; import { FaceOverlay } from '../Overlay/FaceOverlay'; import { rectangleHeight, rectangleWidth, windowHeight, windowWidth, } from '../Overlay/faceConstants'; import RNPhotoManipulator, { MimeType } from 'react-native-photo-manipulator'; import handleCaptureUpload from '../../utils/selfie/handleCaptureUpload'; import { calculateHint, detectFace, loadFaceDetectionModel, type BoundingBox, } from '../../helpers/faceDetection'; import { useSharedValue } from 'react-native-worklets-core'; import { type GetIdSdk } from '../../index'; import * as GetidSdkTypes from '../../types/sdkTypes'; import { GetIDProvider } from '../../contexts/getid-context'; import { scoreFrame, type Scores } from '../../helpers/faceScoring'; import ImagePreview from '../ImagePreview'; import handleCapturePreview, { type PreviewImage, } from '../../utils/documentScan/handleCapturePreview'; type SelfieScanProps = { sdk: GetIdSdk; } & InitialiseSelfieProps; const SelfieScan = React.forwardRef((props: SelfieScanProps, ref) => { useEffect(() => { Logger.log('SelfieScan component mounted'); }, []); const screenSize = useWindowDimensions(); const [appDimension, setAppDimensions] = React.useState({ width: screenSize.width, height: screenSize.height, }); const [cameraDimension, setCameraDimension] = React.useState<{ width: number; height: number; x: number; y: number; }>(); const [frameTimestamp, setFrameTimestamp] = React.useState(0); const updateDataFromWorklet = Worklets.createRunOnJS(setFrameTimestamp); const detectionResults = useSharedValue<{ results: BoundingBox[]; hint: GetidSdkTypes.SelfieUserHint; scores?: Scores | null; } | null>(null); const [captureStarted, setCaptureStarted] = React.useState(false); const [appHeight, setAppHeight] = React.useState(windowHeight); const [frameStatus, setFrameStatus] = React.useState('neutral'); const [previewImage, setPreviewImage] = React.useState(null); const [isReady, setIsReady] = React.useState(false); const [model, setModel] = React.useState(null); const [orientation] = React.useState<'portrait' | 'landscape'>('portrait'); const [isTakingPicture, setIsTakingPicture] = React.useState(false); const [isLoading, setIsLoading] = React.useState(false); const [previewLoaded, setPreviewLoaded] = React.useState(false); const { cameraRef, hasPermission, device } = useCameraSetup({ type: 'front', }); const capturePromise = useRef({ resolve: (_data: any) => {}, reject: (_data: any) => {}, }); const updateLayout = (event: LayoutChangeEvent) => { const appDimensions = { width: event.nativeEvent.layout.width, height: event.nativeEvent.layout.height, }; setAppHeight(event.nativeEvent.layout.height); setAppDimensions(appDimensions); }; const updateCamDimensions = (e: LayoutChangeEvent) => { const { x, y, width, height } = e.nativeEvent.layout; setCameraDimension({ width, height, x, y }); }; const internalTakePicture = async () => { if (isTakingPicture) return; if (!cameraRef.current) { return capturePromise.current?.reject( new SelfieError('CameraRef is not initialized') ); } setIsTakingPicture(true); setIsLoading(true); const { image, path: imagePath } = (await handleCapturePreview({ cameraRef, })) as PreviewImage; setPreviewImage(imagePath); try { const cropPadding = 250; const widthRatio = image.width / appDimension.width; const heightRatio = image.height / appDimension.height; const cropWidth = Math.min( rect.width * widthRatio + cropPadding, image.width ); const cropHeight = Math.min( rect.height * heightRatio + cropPadding, image.height ); const cropLeft = Math.max(rect.left * widthRatio - cropPadding / 2, 0); const cropTop = Math.max(rect.top * heightRatio - cropPadding / 2, 0); const croppingDimensions = { width: cropWidth, height: cropHeight, left: cropLeft, top: cropTop, }; const MAX_WIDTH = 500; const aspectRatio = croppingDimensions.width / croppingDimensions.height; const MAX_HEIGHT = MAX_WIDTH / aspectRatio; const targetSize = { width: Math.min(croppingDimensions.width, MAX_WIDTH), height: Math.min(croppingDimensions.height, MAX_HEIGHT), }; const cropRegion = { x: croppingDimensions.left, y: croppingDimensions.top, width: croppingDimensions.width, height: croppingDimensions.height, }; const resized = await RNPhotoManipulator.crop( imagePath, cropRegion, targetSize, MimeType.JPEG ); props.onServerValidationStarted?.(); const scannedSelfie = await handleCaptureUpload({ imageUri: resized, sdk: props.sdk, }); setIsLoading(false); if (scannedSelfie.result) { capturePromise.current?.resolve(undefined); } else { setIsTakingPicture(false); return capturePromise.current?.reject( new SelfieError( 'Unsuccessful selfie scan.', scannedSelfie.conclusion as SelfieErrorCode ) ); } } catch (e: any) { console.error('Error:', e); setIsTakingPicture(false); setIsLoading(false); setPreviewImage(null); return capturePromise.current?.reject(new SelfieError(e.message)); } }; const takePicture = async () => { return internalTakePicture(); }; const wDiff = (windowWidth - rectangleWidth) / 2; const hDiff = (appHeight - rectangleHeight) / 2; const rect = { centerW: windowWidth / 2, centerH: appHeight / 2, width: rectangleWidth, height: rectangleHeight, top: hDiff, left: wDiff, right: windowWidth - wDiff, bottom: appHeight - hDiff, }; const capture = () => { setPreviewImage(null); return new Promise((resolve, reject) => { Logger.log('Starting Capture process'); capturePromise.current = { resolve, reject, }; if (!device) { Logger.log('Camera device not found'); return capturePromise.current.reject( new SelfieError( 'Camera device not found', GetidSdkTypes.GeneralErrorCode.NO_CAMERA_DEVICE ) ); } if (!hasPermission) { Logger.log('Camera permission not granted'); return capturePromise.current.reject( new SelfieError( 'Camera permission not granted', GetidSdkTypes.GeneralErrorCode.NO_CAMERA_PERMISSION ) ); } setCaptureStarted(true); }); }; const stopCapture = () => { Logger.log('Stopping Capture process'); setCaptureStarted(false); }; const handleFaceDetection = () => { if (detectionResults.value?.hint) { props.onUserHintChange?.(detectionResults.value.hint); } const isSuccess = detectionResults.value?.hint === GetidSdkTypes.SelfieUserHint.SELFIE_OK; setFrameStatus(isSuccess ? 'success' : 'neutral'); if (isSuccess && props.useAutoCapture) internalTakePicture(); }; React.useImperativeHandle( ref, () => ({ capture, stopCapture, takePicture, }) as SelfieScanHandle ); useEffect(() => { const loadModel = async () => { try { const dtModel = await loadFaceDetectionModel(); setModel(dtModel); } catch (e) { console.error('Error:', e); } }; loadModel(); }, []); useEffect(() => { handleFaceDetection(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [frameTimestamp]); const { resize } = useResizePlugin(); const getImageRotate = (frameOrientation: string) => { 'worklet'; switch (frameOrientation) { case 'portrait': return 0; case 'landscape-left': return Platform.OS === 'ios' ? 0 : 90; case 'landscape-right': return 270; case 'portrait-upside-down': return 180; default: return 0; } }; const frameProcessor = useFrameProcessor( (_frame: any) => { 'worklet'; if (isTakingPicture) return; if (!captureStarted || !isReady) return; runAtTargetFps(1, () => { if (!model) { console.log('Model not loaded'); return; } // on IOS, always set isPortrait to false const isPortrait = Platform.OS !== 'ios' && orientation === 'portrait'; const rotation = `${getImageRotate(_frame.orientation)}deg` as | '0deg' | '90deg' | '180deg' | '270deg'; const resized = resize(_frame, { rotation, pixelFormat: 'rgb', dataType: 'float32', scale: { width: 128, height: 128, }, }); const originalSize = { width: isPortrait ? _frame.height : _frame.width, height: isPortrait ? _frame.width : _frame.height, }; const results = detectFace(model, resized); const scaledScreenSize: { width: number; height: number } = { width: appDimension.width * screenSize.scale, height: appDimension.height * screenSize.scale, }; const _frameScreenScale: Point = { x: originalSize.width / scaledScreenSize.width, y: originalSize.height / scaledScreenSize.height, }; const _frameSquareDiff = Math.abs( originalSize.width - originalSize.height ); const screenSquareDiff = _frameSquareDiff / _frameScreenScale.y; const scale: Point = { x: originalSize.width / _frameScreenScale.y, y: (cameraDimension?.height as number) * screenSize.scale - screenSquareDiff, }; const diff: Point = { x: (originalSize.width / _frameScreenScale.y - scaledScreenSize.width) / 2, y: screenSquareDiff / 2, }; const recalculatePoint = (point: Point): Point => { const newPoint = { ...point }; const kX = _frame.isMirrored ? newPoint.x : 1 - newPoint.x; newPoint.x = kX * scale.x - diff.x; newPoint.y = newPoint.y * scale.y + diff.y; newPoint.x /= screenSize.scale; newPoint.y /= screenSize.scale; return newPoint; }; for (let i = 0; i < results.length; i++) { const faceDetection = results[i] as BoundingBox; faceDetection.tl = recalculatePoint(faceDetection.tl); faceDetection.br = recalculatePoint(faceDetection.br); faceDetection.width = Math.abs( faceDetection.br.x - faceDetection.tl.x ); faceDetection.landmarks.nose = recalculatePoint( faceDetection.landmarks.nose ); faceDetection.landmarks.leftEye = recalculatePoint( faceDetection.landmarks.leftEye ); faceDetection.landmarks.rightEye = recalculatePoint( faceDetection.landmarks.rightEye ); faceDetection.landmarks.leftEar = recalculatePoint( faceDetection.landmarks.leftEar ); faceDetection.landmarks.rightEar = recalculatePoint( faceDetection.landmarks.rightEar ); faceDetection.landmarks.mouth = recalculatePoint( faceDetection.landmarks.mouth ); faceDetection.height = Math.abs( faceDetection.br.y - faceDetection.tl.y ); faceDetection.area = faceDetection.width * faceDetection.height; } let scores = null; if (props.useAutoCapture) { const sequence = detectionResults.value?.results ? [{ ...detectionResults.value.results }, results] : [results]; scores = scoreFrame(sequence); } const hint = calculateHint(results, screenSize, rect, scores); detectionResults.value = { results, hint, scores, }; updateDataFromWorklet(new Date().valueOf()); }); }, [ model, captureStarted, isReady, rect, isTakingPicture, props.useAutoCapture, ] ); const setReadyState = () => { setTimeout(() => { setIsReady(true); setTimeout(() => { if (props.onCameraReady) { props.onCameraReady(); } }, 400); }, 50); }; return ( {device && hasPermission && ( setReadyState()} onLayout={updateCamDimensions} ref={cameraRef} device={device} isActive={true} frameProcessor={frameProcessor} style={CameraStyles.camera} photo={true} video={true} photoHdr={false} photoQualityBalance="speed" orientation="portrait" exposure={0} /> )} {previewImage && ( { setTimeout(() => { setPreviewLoaded(true); }, 320); }} /> )} {isReady && ( )} ); }); export default memo(SelfieScan);