/* * Copyright 2019 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.evetensorflowyolo; import android.app.Activity; import android.graphics.Bitmap; import android.graphics.Bitmap.Config; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Matrix; import android.graphics.Paint; import android.graphics.Paint.Style; import android.graphics.RectF; import android.graphics.Typeface; import android.media.ImageReader.OnImageAvailableListener; import android.os.SystemClock; import android.util.Log; import android.util.Size; import android.util.TypedValue; import android.view.View; import android.widget.ArrayAdapter; import android.widget.ListView; import android.widget.Toast; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import com.evetensorflowyolo.customview.OverlayView; import com.evetensorflowyolo.customview.OverlayView.DrawCallback; import com.evetensorflowyolo.customview.RecognitionScoreView; import com.evetensorflowyolo.env.BorderedText; import com.evetensorflowyolo.env.ImageUtils; import com.evetensorflowyolo.env.Logger; import com.evetensorflowyolo.tflite.Classifier; import com.evetensorflowyolo.tflite.DetectorFactory; import com.evetensorflowyolo.tflite.YoloV5Classifier; import com.evetensorflowyolo.ResultsActivity; import com.evetensorflowyolo.tracking.MultiBoxTracker; /** * An activity that uses a TensorFlowMultiBoxDetector and ObjectTracker to detect and then track * objects. */ public class DetectorActivity2 extends CameraActivity implements OnImageAvailableListener { private static final Logger LOGGER = new Logger(); private static final DetectorMode MODE = DetectorMode.TF_OD_API; private static final float MINIMUM_CONFIDENCE_TF_OD_API = 0.3f; private static final boolean MAINTAIN_ASPECT = true; private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 640); private static final boolean SAVE_PREVIEW_BITMAP = false; private static final float TEXT_SIZE_DIP = 10; OverlayView trackingOverlay; private Integer sensorOrientation; private YoloV5Classifier detector; private long lastProcessingTimeMs; private Bitmap rgbFrameBitmap = null; private Bitmap croppedBitmap = null; //make static so accessible in results_activity static Bitmap cropCopyBitmap = null; static Bitmap displayBitmap = null; private boolean computingDetection = false; private long timestamp = 0; private Matrix frameToCropTransform; private Matrix cropToFrameTransform; private MultiBoxTracker tracker; private BorderedText borderedText; //create a file to save output //File file = new File(getExternalFilesDir(null), "detections.jpg"); @Override public void onPreviewSizeChosen(final Size size, final int rotation) { Log.v("Love", "THis is detectoractuvy2"); Log.v("love", "I'm back"); final float textSizePx = TypedValue.applyDimension( TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); Log.v("Love", "got textsizepx"); borderedText = new BorderedText(textSizePx); Log.v("Love", "got bordered text"); borderedText.setTypeface(Typeface.MONOSPACE); Log.v("Love", "set typeface"); tracker = new MultiBoxTracker(this); Log.v("Love", "got tracker"); final int modelIndex = modelView.getCheckedItemPosition(); Log.v("Love", "got modelindex"); final String modelString = (String) modelStrings.get(modelIndex); Log.v("Love", "got modelstring"); try { detector = DetectorFactory.getDetector(getAssets(), modelString); Log.v("Love", "got detector"); } catch (final IOException e) { e.printStackTrace(); LOGGER.e(e, "Exception initializing classifier!"); Toast toast = Toast.makeText( getApplicationContext(), "Classifier could not be initialized", Toast.LENGTH_SHORT); toast.show(); finish(); } int cropSize = detector.getInputSize(); Log.v("Love", "got cropsize"); previewWidth = size.getWidth(); Log.v("Love", "got width"); previewHeight = size.getHeight(); Log.v("Love", "got height"); sensorOrientation = rotation - getScreenOrientation(); Log.v("Love", "got sensorOrient"); LOGGER.i("Camera orientation relative to screen canvas: %d", sensorOrientation); Log.v("Love", "logger cam orient"); LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight); rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888); Log.v("Love", "got rgbbitmap"); croppedBitmap = Bitmap.createBitmap(cropSize, cropSize, Config.ARGB_8888); Log.v("Love", "got cropped bitmap"); frameToCropTransform = ImageUtils.getTransformationMatrix( previewWidth, previewHeight, cropSize, cropSize, sensorOrientation, MAINTAIN_ASPECT); Log.v("Love", "got frametocroptransform"); cropToFrameTransform = new Matrix(); Log.v("Love", "got ne w matricx"); frameToCropTransform.invert(cropToFrameTransform); Log.v("Love", "got invert"); trackingOverlay = (OverlayView) findViewById(R.id.tracking_overlay); Log.v("Love", "found overlay"); trackingOverlay.addCallback( new DrawCallback() { @Override public void drawCallback(final Canvas canvas) { //don't draw rectangles on pics tracker.draw(canvas, true); if (isDebug()) { tracker.drawDebug(canvas); } } }); Log.v("Love", "added call back tracker"); tracker.setFrameConfiguration(previewWidth, previewHeight, sensorOrientation); Log.v("Love", "set tracker frame config"); /*resultListView = findViewById(R.id.detections); resultStrings.add("Love"); ArrayAdapter resultsAdapter = new ArrayAdapter<>( DetectorActivity2.this , R.layout.resultview_row, R.id.resultview_row_text, resultStrings); resultListView.setAdapter(resultsAdapter); */ } protected void updateActiveModel() { // Get UI information before delegating to background resultStrings.clear(); final int modelIndex = modelView.getCheckedItemPosition(); Log.v("Love", "got modelIndex"); final int deviceIndex = deviceView.getCheckedItemPosition(); Log.v("Love", "got device index"); String threads = threadsTextView.getText().toString().trim(); Log.v("Love", "got threads"); final int numThreads = Integer.parseInt(threads); handler.post(() -> { if (modelIndex == currentModel && deviceIndex == currentDevice && numThreads == currentNumThreads) { return; } currentModel = modelIndex; currentDevice = deviceIndex; currentNumThreads = numThreads; // Disable classifier while updating if (detector != null) { detector.close(); detector = null; } // Lookup names of parameters. String modelString = (String) modelStrings.get(modelIndex); String device = (String) deviceStrings.get(deviceIndex); LOGGER.i("Changing model to " + modelString + " device " + device); // Try to load model. try { detector = DetectorFactory.getDetector(getAssets(), modelString); // Customize the interpreter to the type of device we want to use. if (detector == null) { return; } } catch(IOException e) { e.printStackTrace(); LOGGER.e(e, "Exception in updateActiveModel()"); Toast toast = Toast.makeText( getApplicationContext(), "Classifier could not be initialized", Toast.LENGTH_SHORT); toast.show(); finish(); } if (device.equals("CPU")) { detector.useCPU(); } else if (device.equals("GPU")) { detector.useGpu(); } else if (device.equals("NNAPI")) { detector.useNNAPI(); } detector.setNumThreads(numThreads); int cropSize = detector.getInputSize(); croppedBitmap = Bitmap.createBitmap(cropSize, cropSize, Config.ARGB_8888); frameToCropTransform = ImageUtils.getTransformationMatrix( previewWidth, previewHeight, cropSize, cropSize, sensorOrientation, MAINTAIN_ASPECT); cropToFrameTransform = new Matrix(); frameToCropTransform.invert(cropToFrameTransform); }); } @Override protected void processImage() { ++timestamp; final long currTimestamp = timestamp; trackingOverlay.postInvalidate(); // No mutex needed as this method is not reentrant. if (computingDetection) { readyForNextImage(); return; } computingDetection = true; LOGGER.i("Preparing image " + currTimestamp + " for detection in bg thread."); rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight); readyForNextImage(); final Canvas canvas = new Canvas(croppedBitmap); canvas.drawBitmap(rgbFrameBitmap, frameToCropTransform, null); // For examining the actual TF input. if (SAVE_PREVIEW_BITMAP) { ImageUtils.saveBitmap(croppedBitmap); } runInBackground( new Runnable() { @Override public void run() { LOGGER.i("Running detection on image " + currTimestamp); final long startTime = SystemClock.uptimeMillis(); final List results = detector.recognizeImage(croppedBitmap); lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime; Log.e("CHECK", "run: " + results.size()); cropCopyBitmap = Bitmap.createBitmap(croppedBitmap); final Canvas canvas = new Canvas(cropCopyBitmap); final Paint paint = new Paint(); paint.setColor(Color.RED); paint.setStyle(Style.STROKE); paint.setStrokeWidth(2.0f); float minimumConfidence = MINIMUM_CONFIDENCE_TF_OD_API; switch (MODE) { case TF_OD_API: minimumConfidence = MINIMUM_CONFIDENCE_TF_OD_API; break; } final List mappedRecognitions = new LinkedList(); //iterating through results for (final Classifier.Recognition result : results) { final RectF location = result.getLocation(); String info = ""; if (location != null && result.getConfidence() >= minimumConfidence) { canvas.drawRect(location, paint); cropToFrameTransform.mapRect(location); result.setLocation(location); mappedRecognitions.add(result); info = result.getTitle(); //resultsStrings.add("HI"); if (!resultStrings.contains(info)) { //get all distinct reads above //can add threshold here resultStrings.add(info); Log.v("Love", info); } } } tracker.trackResults(mappedRecognitions, currTimestamp); //call tracker.debug and using my own bitmap and canvas //maybe a second copy of cropped--ie. cretea bitMap from rectangle one //now have bitmap, create new canvas..w/my copy passed in //call debug method, passing in that canvas displayBitmap = Bitmap.createBitmap(cropCopyBitmap); final Canvas displayCanvas = new Canvas(displayBitmap); //tracker.drawDebug(displayCanvas); //create instance of //RecognitionScoreView.onDraw(displayCanvas); //is this my overlay //trackingOverlay.draw(displayCanvas); tracker.draw(displayCanvas, false); trackingOverlay.postInvalidate(); //pass in file object and bitmap //ImageUtils.saveBitmap(cropCopyBitmap, file); //displayView.setImageBitmap(cropCopyBitmap); computingDetection = false; runOnUiThread( new Runnable() { @Override public void run() { showFrameInfo(previewWidth + "x" + previewHeight); showCropInfo(cropCopyBitmap.getWidth() + "x" + cropCopyBitmap.getHeight()); showInference(lastProcessingTimeMs + "ms"); } }); } }); } @Override protected int getLayoutId() { return R.layout.tfe_od_camera_connection_fragment_tracking; } @Override protected Size getDesiredPreviewFrameSize() { return DESIRED_PREVIEW_SIZE; } // Which detection model to use: by default uses Tensorflow Object Detection API frozen // checkpoints. private enum DetectorMode { TF_OD_API; } @Override protected void setUseNNAPI(final boolean isChecked) { runInBackground(() -> detector.setUseNNAPI(isChecked)); } @Override protected void setNumThreads(final int numThreads) { runInBackground(() -> detector.setNumThreads(numThreads)); } }