import { Platform, View, Text, Image, useWindowDimensions } from 'react-native'; import React, { memo, useCallback, useEffect, useMemo, useRef } from 'react'; import { Camera, runAtTargetFps, useFrameProcessor, type Frame, type Point, } from 'react-native-vision-camera'; import { launchImageLibrary } from 'react-native-image-picker'; import Logger from '../../utils/logger'; import handleCapturePreview, { type PreviewImage, } from '../../utils/documentScan/handleCapturePreview'; import { DocumentConclusion, DocumentScanError, type DocumentScanHandle, DocumentScanUserHint, type ScannedDocument, } from '../../types/sdkTypes'; import RNPhotoManipulator, { MimeType } from 'react-native-photo-manipulator'; import { StyleSheet } from 'react-native'; import ImagePreview from '../ImagePreview'; import { type GetIdSdk } from '../../index'; import * as GetidSdkTypes from '../../types/sdkTypes'; import verifyApplication from '../../api/verifyApplication'; import handleTakePicture from '../../utils/documentScan/handleTakePicture'; import { DocumentOverlay, Quadrangles } from '../Overlay/DocumentOverlay'; import { getRectangleDimensions } from '../Overlay/utils'; import { useCameraSetup } from '../../hooks'; import CameraStyles from '../../styles/CameraStyles'; import type { LayoutChangeEvent } from 'react-native/Libraries/Types/CoreEventTypes'; import type { TensorflowModel } from '@get-id/react-native-fast-tflite'; import { type DocumentPrediction } from '../../helpers/documentDetection'; import { detectDocument, loadDocumentDetectionModel, loadQualityCheckModel, } from '../../helpers/documentDetectionTF'; import { DocumentDetector, TrackedDocumentStatus, mean, quadrangleArea, getPredictionCrop, quadrangleMovement, type DocumentCorners, type DocumentDetectorResult, TrackedDocument, } from '../../helpers/documentDetector'; import { useResizePlugin } from 'vision-camera-resize-plugin'; import { useSharedValue, Worklets } from 'react-native-worklets-core'; import { GetIDProvider } from '../../contexts/getid-context'; import { getImageRotationToResize, getPointConverter, } from '../../helpers/camera'; import { getDebugMode, fullDebugMode } from '../../helpers/debug'; import { DebugContext } from '../../contexts/debug-context'; export type DocumentStepConfig = { allowFromGallery: boolean; }; const SHIFT_THRESHOLD = 0.05; // keep cooldown long enough to avoid model flickering const GLARE_COOLDOWN = 1100; type DocumentCrop = { index: number; crop: Float32Array; // base64 encoded image debugCrop?: string; }; type DocumentScanProps = { onUserHintChange?: (hint: DocumentScanUserHint) => void; onDocumentScanServerValidationStarted?: () => void; useAutoCapture?: boolean; debugMode?: boolean; sdk: GetIdSdk; docType: 'front' | 'back'; config: DocumentStepConfig; onCameraReady?: () => void; }; const outOfFrame = { top: false, bottom: false, left: false, right: false, }; const DocumentScan = React.forwardRef((props: DocumentScanProps, ref) => { const debugHelper = React.useContext(DebugContext); useEffect(() => { Logger.log('DocumentScan component mounted'); }, []); const debugMode = useMemo(() => { if (props.debugMode) { return fullDebugMode; } return getDebugMode(); }, [props.debugMode]); const screenSize = useWindowDimensions(); const [appDimension, setAppDimensions] = React.useState({ width: screenSize.width, height: screenSize.height, }); const [qualityCheckModel, setQualityCheckModel] = React.useState(null); const [model, setModel] = React.useState(null); const [documentDetector, setDocumentDetector] = React.useState(null); const detectionResults = useSharedValue([]); const [detectionStatuses, setDetectionStatuses] = React.useState< TrackedDocumentStatus[] >([]); const [documentCrops, setDocumentCrops] = React.useState([]); const lastGlareTimestamp = useSharedValue(0); const [captureStarted, setCaptureStarted] = React.useState(false); const [prevCaptureStarted, setPrevCaptureStarted] = React.useState(false); const [previewImage, setPreviewImage] = React.useState(null); const [isSaving, setIsSaving] = React.useState(false); const [isLoading, setIsLoading] = React.useState(false); const [isReady, setIsReady] = React.useState(false); const [previewLoaded, setPreviewLoaded] = React.useState(false); const capturePromise = useRef<{ resolve: (value?: any) => void; reject: (reason?: any) => void; }>(); const { cameraRef, hasPermission, device } = useCameraSetup({ type: 'back' }); const [cameraResizeMode] = React.useState<'cover' | 'contain'>('cover'); const [orientation] = React.useState<'portrait' | 'landscape'>('portrait'); const [fromUpload, setFromUpload] = React.useState(false); const { width: rectangleWidth, height: rectangleHeight } = getRectangleDimensions(); const [cameraDimension, setCameraDimension] = React.useState<{ width: number; height: number; x: number; y: number; }>(); const [inFrame, setInFrame] = React.useState(outOfFrame); // TODO: make dimensions dynamic const overlayRect = useMemo(() => { const width = rectangleWidth; const height = rectangleHeight; const top = (appDimension.height - height) / 2; const left = (appDimension.width - width) / 2; const right = appDimension.width - (appDimension.width - width) / 2; const bottom = appDimension.height - (appDimension.height - height) / 2; return { height, width, top, left, right, bottom, }; }, [ appDimension.height, appDimension.width, rectangleHeight, rectangleWidth, ]); const updateLayout = (event: LayoutChangeEvent) => { const appDimensions = { width: event.nativeEvent.layout.width, height: event.nativeEvent.layout.height, }; setAppDimensions(appDimensions); }; const updateCamDimensions = (e: LayoutChangeEvent) => { const { x, y, width, height } = e.nativeEvent.layout; setCameraDimension({ width, height, x, y }); }; const reset = () => { setPreviewImage(null); setPreviewLoaded(false); setIsSaving(false); setIsLoading(false); }; const clearPreviewIfIssues = (scannedDoc: ScannedDocument) => { if ( (props.docType === 'front' && scannedDoc.conclusion === DocumentConclusion.BACK_SIDE_MISSING) || scannedDoc.conclusion === DocumentConclusion.OK ) { return; } setCaptureStarted(prevCaptureStarted); reset(); }; const takePicture = async () => { try { setIsSaving(true); setIsLoading(true); const { path: previewImageUri, image } = (await handleCapturePreview({ cameraRef, })) as PreviewImage; if (typeof previewImageUri === 'string') { setFromUpload(false); setPreviewImage(previewImageUri); } const cropPadding = 100; const widthRatio = image.width / appDimension.width; const heightRatio = image.height / appDimension.height; const cropWidth = Math.min( overlayRect.width * widthRatio + cropPadding, image.width ); const cropHeight = Math.min( overlayRect.height * heightRatio + cropPadding, image.height ); const cropLeft = Math.max( overlayRect.left * widthRatio - cropPadding / 2, 0 ); const cropTop = Math.max( overlayRect.top * heightRatio - cropPadding / 2, 0 ); const croppingDimensions = { width: cropWidth, height: cropHeight, left: cropLeft, top: cropTop, }; const MAX_WIDTH = 1000; const aspectRatio = croppingDimensions.width / croppingDimensions.height; const MAX_HEIGHT = MAX_WIDTH / aspectRatio; const targetSize = { width: Math.min(croppingDimensions.width, MAX_WIDTH), height: Math.min(croppingDimensions.height, MAX_HEIGHT), }; const cropRegion = { x: croppingDimensions.left, y: croppingDimensions.top, width: croppingDimensions.width, height: croppingDimensions.height, }; const resized = await RNPhotoManipulator.crop( previewImageUri, cropRegion, targetSize, MimeType.JPEG ); if (props.onDocumentScanServerValidationStarted) { props.onDocumentScanServerValidationStarted(); } const scannedDoc = await handleTakePicture({ previewImageUri: resized as string, sdk: props.sdk, docType: props.docType, }); setIsLoading(false); capturePromise.current?.resolve(scannedDoc); setCaptureStarted(prevCaptureStarted); props.sdk.tracking.trySendEvent(`${props.docType}-preview`, 'completed'); } catch (e: any) { setPreviewImage(null); setPreviewLoaded(false); setIsSaving(false); setIsLoading(false); capturePromise.current?.reject(new DocumentScanError(e.message)); } }; const capture = async () => { Logger.log('Starting Capture process'); props.sdk.tracking.trySendEvent(`${props.docType}-preview`, 'started'); reset(); setCaptureStarted(true); setPrevCaptureStarted(true); return new Promise((resolve, reject) => { capturePromise.current = { resolve, reject }; if (!device) { Logger.log('Camera device not found'); const errorCode = GetidSdkTypes.GeneralErrorCode.NO_CAMERA_DEVICE; props.sdk.tracking.trySendEvent('error', 'started', { errorCode }); return capturePromise.current.reject( new DocumentScanError('Camera device not found', errorCode) ); } if (!hasPermission) { Logger.log('Camera permission not granted'); const errorCode = GetidSdkTypes.GeneralErrorCode.NO_CAMERA_PERMISSION; props.sdk.tracking.trySendEvent('error', 'started', { errorCode }); return capturePromise.current.reject( new DocumentScanError('Camera permission not granted', errorCode) ); } }); }; const stopCapture = async () => { Logger.log('Stopping Capture process'); setCaptureStarted(false); setPrevCaptureStarted(false); }; const uploadDocument = async () => { Logger.log('Uploading document...'); if (!props.config.allowFromGallery) { throw new DocumentScanError('Uploading from gallery is not allowed'); } props.sdk.tracking.trySendEvent( `${props.docType}-side-from-gallery`, 'started' ); props.sdk.tracking.trySendEvent(`${props.docType}-preview`, 'started'); try { const response = await launchImageLibrary({ mediaType: 'photo', selectionLimit: 1, }); console.log('pickedFile', response); const uri = response.assets?.[0]?.uri; if (!uri) { throw new DocumentScanError('No file selected'); } if (typeof uri === 'string') { setCaptureStarted(false); setFromUpload(true); setPreviewImage(uri); } if (props.onDocumentScanServerValidationStarted) { props.onDocumentScanServerValidationStarted(); } try { const scannedDoc = await handleTakePicture({ previewImageUri: uri, sdk: props.sdk, docType: props.docType, }); props.sdk.tracking.trySendEvent( `${props.docType}-side-from-gallery`, 'completed' ); props.sdk.tracking.trySendEvent( `${props.docType}-preview`, 'completed' ); clearPreviewIfIssues(scannedDoc); return scannedDoc; } catch (e: any) { setPreviewImage(null); setPreviewLoaded(false); setCaptureStarted(prevCaptureStarted); throw new DocumentScanError(e.message); } } catch (error: any) { setPreviewImage(null); setPreviewLoaded(false); setCaptureStarted(prevCaptureStarted); throw new DocumentScanError(error.message); } }; const submitForVerification = async () => { const result = await verifyApplication({ sdk: props.sdk }); return result; }; React.useImperativeHandle( ref, () => ({ takePicture, capture, stopCapture, uploadDocument, submitForVerification, }) as DocumentScanHandle ); useEffect(() => { const loadModel = async () => { const dtModel = await loadDocumentDetectionModel(); setModel(dtModel); setDocumentDetector( new DocumentDetector({ frameSize: { width: 160, height: 160 }, // TODO: tweak these values documentTrackTimeout: 1500, documentTrackSpeedThreshold: 0.04, documentMinFrames: 2, documentMaxMovementSpeed: 0.00007, documentMaxRotation: 0.1, documentMinFrameGap: 0.05, documentMinCloseToCamera: 0.3, documentMaxCloseToCamera: 0.95, documentMinCloseToCenter: 0.0, documentMaxSkewAngle: 0.1, outOfCameraFrameThreshold: 0.5, }) ); const qcModel = await loadQualityCheckModel(); setQualityCheckModel(qcModel); }; loadModel(); }, [props.useAutoCapture]); const getUserHint = useCallback( ( timestamp: number, detectorResults: DocumentDetectorResult, convertPoint: (point: Point) => Point, qualityChecks: { glare?: boolean } = {} ): DocumentScanUserHint => { if (detectorResults.trackedDocuments.length > 2) { setInFrame(outOfFrame); return DocumentScanUserHint.MANY_DOCUMENTS; } function convertPoints(doc: TrackedDocument) { const lastPrediction = doc.timeline[doc.timeline.length - 1]!; const points = lastPrediction.prediction.points; return points.map(convertPoint) as DocumentCorners; } const rescaledTrackedDocuments = detectorResults.trackedDocuments.map(convertPoints); const expectedQuadrangle: DocumentCorners = [ { x: overlayRect.left, y: overlayRect.top }, { x: overlayRect.right, y: overlayRect.top }, { x: overlayRect.right, y: overlayRect.bottom }, { x: overlayRect.left, y: overlayRect.bottom }, ]; const distances = rescaledTrackedDocuments.map((points) => { return quadrangleMovement(points, expectedQuadrangle); }); const closestDocumentIndex = distances.reduce( (acc, curr, i) => (curr < distances[acc]! ? i : acc), 0 ); const rescaledDocument = rescaledTrackedDocuments[closestDocumentIndex]; const document = detectorResults.trackedDocuments[closestDocumentIndex]; if (!document || !rescaledDocument) { setInFrame(outOfFrame); return DocumentScanUserHint.NO_DOCUMENT; } if (document.status === TrackedDocumentStatus.NOT_ENOUGH_FRAMES) { setInFrame(outOfFrame); return DocumentScanUserHint.NOTHING; } const sortedX = rescaledDocument.map((p) => p.x).sort((a, b) => a - b); const sortedY = rescaledDocument.map((p) => p.y).sort((a, b) => a - b); const meanTop = mean(sortedY.slice(0, 2)); const meanBottom = mean(sortedY.slice(2)); const meanLeft = mean(sortedX.slice(0, 2)); const meanRight = mean(sortedX.slice(2)); const offTop = overlayRect.top - meanTop; const offBottom = meanBottom - overlayRect.bottom; const offLeft = overlayRect.left - meanLeft; const offRight = meanRight - overlayRect.right; const offTopRatio = offTop / overlayRect.width; const offBottomRatio = offBottom / overlayRect.width; const offLeftRatio = offLeft / overlayRect.width; const offRightRatio = offRight / overlayRect.width; const newInFrame = { top: offTopRatio < SHIFT_THRESHOLD, bottom: offBottomRatio < SHIFT_THRESHOLD, left: offLeftRatio < SHIFT_THRESHOLD, right: offRightRatio < SHIFT_THRESHOLD, }; setInFrame(newInFrame); const mapPriorityStatuses = new Map([ [ TrackedDocumentStatus.OUT_OF_CAMERA_FRAME, DocumentScanUserHint.OUT_OF_CAMERA_FRAME, ], [TrackedDocumentStatus.TOO_MUCH_ROTATION, DocumentScanUserHint.ROTATE], [TrackedDocumentStatus.TOO_CLOSE, DocumentScanUserHint.MOVE_BACK], [TrackedDocumentStatus.TOO_FAR, DocumentScanUserHint.MOVE_FORWARD], ]); const mappedPriorityStatus = mapPriorityStatuses.get(document.status); if (mappedPriorityStatus) { return mappedPriorityStatus; } const areaRatio = quadrangleArea(rescaledDocument) / quadrangleArea(expectedQuadrangle); const areaRatioSqrt = Math.sqrt(areaRatio); if (areaRatioSqrt < 1.0) { const diff = 1.0 - areaRatioSqrt; if (diff > 0.13) { return DocumentScanUserHint.MOVE_FORWARD; } } else { const diff = areaRatioSqrt - 1.0; if (diff > 0.1) { return DocumentScanUserHint.MOVE_BACK; } } const offMaxRatio = Math.max( offTopRatio, offBottomRatio, offLeftRatio, offRightRatio ); if (offMaxRatio > SHIFT_THRESHOLD) { const moveHints = new Map([ [offTopRatio, DocumentScanUserHint.MOVE_UP], [offBottomRatio, DocumentScanUserHint.MOVE_DOWN], [offLeftRatio, DocumentScanUserHint.MOVE_LEFT], [offRightRatio, DocumentScanUserHint.MOVE_RIGHT], ]); const hint = moveHints.get(offMaxRatio); if (hint) { return hint; } } const mapStatuses = new Map([ [TrackedDocumentStatus.TOO_SKEWED, DocumentScanUserHint.SKEWED], [ TrackedDocumentStatus.TOO_FAST_MOVEMENT, DocumentScanUserHint.HOLD_STILL, ], ]); const mappedStatus = mapStatuses.get(document.status); if (mappedStatus) { return mappedStatus; } const glare = qualityChecks.glare || timestamp - lastGlareTimestamp.value < GLARE_COOLDOWN; return glare ? DocumentScanUserHint.GLARE : DocumentScanUserHint.POSITION_OK; }, [overlayRect, lastGlareTimestamp] ); const onDocumentDetected = Worklets.createRunOnJS( ( timestamp: number, prediction: DocumentPrediction[], crops: DocumentCrop[], originalFrameSize: { width: number; height: number }, croppedFrameSize: { width: number; height: number }, mirrored: boolean ) => { setDocumentCrops(crops); const convertPoint = getPointConverter( originalFrameSize, appDimension, cameraDimension!, 'back', screenSize.scale, mirrored ); detectionResults.value = prediction.map((doc) => ({ ...doc, points: doc.points.map(convertPoint), })); if (!documentDetector) { console.log('Document detector not loaded'); return { detectorResults: { prediction, trackedDocuments: [] }, }; } const detectorResults = documentDetector.processFrame( timestamp, prediction, croppedFrameSize ); setDetectionStatuses( detectorResults.trackedDocuments.map((d) => d.status) ); return { detectorResults }; } ); const onQualityCheck = Worklets.createRunOnJS( ( timestamp: number, detectorResults: DocumentDetectorResult, originalFrameSize: { width: number; height: number }, mirrored: boolean, qualityChecks: { glare: boolean } ) => { if (qualityChecks.glare) { lastGlareTimestamp.value = timestamp; } const convertPoint = getPointConverter( originalFrameSize, appDimension, cameraDimension!, 'back', screenSize.scale, mirrored ); const userHint = getUserHint( timestamp, detectorResults, convertPoint, qualityChecks ); if (props.onUserHintChange) { props.onUserHintChange(userHint); } if ( userHint === DocumentScanUserHint.POSITION_OK && props.useAutoCapture && debugMode.takePicture ) { takePicture(); } } ); const { resize } = useResizePlugin(); const frameProcessor = useFrameProcessor( (frame: Frame) => { 'worklet'; if (previewImage || isSaving) return; if (!captureStarted && isReady) return; runAtTargetFps(2, () => { if (!model) { console.log('Model not loaded'); return; } if (props.useAutoCapture && !documentDetector) { console.log('Document detector not loaded'); return; } // on IOS, always set isPortrait to false const isPortrait = Platform.OS !== 'ios' && orientation === 'portrait'; const rotation = getImageRotationToResize(frame); // resize without default cropping performs a center-crop const resized = resize(frame, { rotation, pixelFormat: 'rgb', dataType: 'float32', scale: { width: 160, height: 160, }, }); const inputData = resized; const originalSize = { width: isPortrait ? frame.height : frame.width, height: isPortrait ? frame.width : frame.height, }; const croppedSize = { width: Math.min(originalSize.width, originalSize.height), height: Math.min(originalSize.width, originalSize.height), }; const timestamp = performance.now(); const prediction = detectDocument(model, inputData); const crops: DocumentCrop[] = []; for (let index = 0; index < prediction.length; index++) { const crop = getPredictionCrop(prediction[index]!.points, { rotation, frame: originalSize, padding: -0.08, }); const documentCrop = resize(frame, { // rotation doesn't affect the coordinates for cropping rotation, pixelFormat: 'rgb', dataType: 'float32', scale: { width: 80, height: 80, }, // (0, 0) is the top-right corner (for Google Pixel 3a) crop, }); const croppedImage: DocumentCrop = { index, crop: documentCrop, }; crops.push(croppedImage); if (debugMode.showCroppedDocument && debugHelper.cropToBase64) { croppedImage.debugCrop = debugHelper.cropToBase64( frame, crop, rotation ); } } const isMirrored = frame.isMirrored; onDocumentDetected( timestamp, prediction, crops, originalSize, croppedSize, isMirrored ).then( ({ detectorResults, }: { detectorResults: DocumentDetectorResult; }) => { 'worklet'; let maxGlare = 0; if (qualityCheckModel) { // TODO: check only okDocuments for (const documentCrop of crops) { const qualityChecks = qualityCheckModel.runSync([ documentCrop.crop, ]); const glareOutput = qualityChecks[0] as Float32Array; const glareArea = glareOutput[0]; if (typeof glareArea !== 'number') { continue; } if (glareArea > maxGlare) { maxGlare = glareArea; } } } else { console.log('Quality check model not loaded'); } const glare = maxGlare > 0.5; onQualityCheck( timestamp, detectorResults, originalSize, isMirrored, { glare, } ); } ); }); }, [documentDetector, captureStarted, isReady, previewImage, isSaving] ); useEffect(() => { props.sdk.tracking.trySendEvent(props.docType, 'started'); return () => { props.sdk.tracking.trySendEvent(props.docType, 'completed'); }; }, [props.sdk, props.docType]); const setReadyState = () => { setTimeout(() => { setIsReady(true); setTimeout(() => { if (props.onCameraReady) { props.onCameraReady(); } }, 200); }, 50); }; return ( {device && hasPermission && ( setReadyState()} onLayout={updateCamDimensions} ref={cameraRef} device={device} isActive={true} style={CameraStyles.camera} frameProcessor={frameProcessor} resizeMode={cameraResizeMode} photo={true} video orientation="portrait" exposure={0} /> )} {previewImage && ( { setTimeout(() => { setPreviewLoaded(true); }, 320); }} /> )} {isReady && ( )} {debugMode.showDocumentDetectorStatus && ( Detection statuses: {detectionStatuses.map((s) => s).join(', ')} )} {debugMode.showCoordinates && detectionResults.value && ( r.points)} /> )} {debugMode.showCroppedDocument && documentCrops .filter((crop) => crop.debugCrop) .map((crop, index) => ( ))} ); }); const styles = StyleSheet.create({ debugText: { backgroundColor: 'white', position: 'absolute', bottom: 0, }, debugImage: { position: 'absolute', top: 0, right: 0, zIndex: 100, }, }); export default memo(DocumentScan);