DeepLabV3+ Android App

<uses-permission android:name="android.permission.CAMERA" />

 <uses-feature android:name="android.hardware.camera2" />

@Override
  protected NeuralNetwork doInBackground(File... params) {
  final SNPE.NeuralNetworkBuilder builder = new
  SNPE.NeuralNetworkBuilder(mApplicationContext)
  // Allows selecting a runtime order for the network.
  // In the example below use DSP and fall back, in order, to GPU then CPU
  // depending on whether any of the runtimes are available.
  .setRuntimeOrder(DSP, GPU, CPU)
  // Loads a model from DLC file
  .setModel(new File("<model-path>"))
  // Build the network
  network = builder.build();
  }

private final TextureView.SurfaceTextureListener mSurfaceTextureListener
          = new TextureView.SurfaceTextureListener() {
      @Override
      public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) {
         //check runtime permission
  if (ContextCompat.checkSelfPermission(getActivity(), Manifest.permission.CAMERA)
          != PackageManager.PERMISSION_GRANTED) {
      requestCameraPermission();
      return;
  }
  
  CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
  try {
      if (!mCameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
          throw new RuntimeException("Time out waiting to lock camera opening.");
      }
  
  //mCameraId = 1(Front Camera), mCameraId = 0(Rear Camera)
      manager.openCamera(mCameraId, mStateCallback, mBackgroundHandler);
  } catch (CameraAccessException e) {
      e.printStackTrace();
  } catch (InterruptedException e) {
      throw new RuntimeException("Interrupted while trying to lock camera opening.", e);
  } 
  }

@Override
public void onOpened(@NonNull CameraDevice cameraDevice) {
  // This method is called when the camera is opened.  We start camera preview here.
  mCameraOpenCloseLock.release();
  mCameraDevice = cameraDevice;  mPreviewRequestBuilder=mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
mPreviewRequestBuilder.addTarget(surface);
}

private final ImageReader.OnImageAvailableListener mOnImageAvailableListener
          = new ImageReader.OnImageAvailableListener() {
      @Override
      public void onImageAvailable(ImageReader reader) {
          mBackgroundHandler.post(new ImageSaver(reader.acquireNextImage(), mFile));
      }
  };

private CameraCaptureSession.CaptureCallback mCaptureCallback = new CameraCaptureSession.CaptureCallback() {
  @Override
  public void onCaptureCompleted(@NonNull CameraCaptureSession session,
                                 @NonNull CaptureRequest request,
                                 @NonNull TotalCaptureResult result) {
  //Get file path from ImageReader 
  String stringUri = mFile.getAbsolutePath();
  Bitmap mBitmap = BitmapFactory.decodeFile(stringUri);
  }
  }

 private Map<String, FloatTensor> inferenceOnBitmap(Bitmap scaledBitmap) {
      final Map<String, FloatTensor> outputs;
      try {
          // safety check
          if (mNeuralnetwork == null || mInputTensorReused == null || scaledBitmap.getWidth() != getInputTensorWidth() || scaledBitmap.getHeight() != getInputTensorHeight()) {
              Logger.d("SNPEHelper", "No NN loaded, or image size different than tensor size");
              return null;
          }
          // [0.3ms] Bitmap to RGBA byte array (size: 513*513*3 (RGBA..))
          mBitmapToFloatHelper.bitmapToBuffer(scaledBitmap);
          // [2ms] Pre-processing: Bitmap (513,513,4 ints) -> Float Input Tensor (513,513,3 floats)
          final float[] inputFloatsHW3 = mBitmapToFloatHelper.bufferToNormalFloatsBGR();
          if (mBitmapToFloatHelper.isFloatBufferBlack())
              return null;
          mInputTensorReused.write(inputFloatsHW3, 0, inputFloatsHW3.length, 0, 0);
          // [31ms on GPU16, 50ms on GPU] execute the inference
          outputs = mNeuralnetwork.execute(mInputTensorsMap);
      } catch (Exception e) {
          e.printStackTrace();
          Logger.d("SNPEHelper", e.getCause() + "");
          return null;
      }
      return outputs;
  }

public float[] bufferToNormalFloatsBGR() {
      
      final byte[] inputArrayHW4 = mByteBufferHW4.array();
      final int area = mFloatBufferHW3.length / 3;
      long sumG = 0;
      int srcIdx = 0, dstIdx = 0;
      final float inputScale = 0.00784313771874f;
      for (int i = 0; i < area; i++) {
          // NOTE: the 0xFF a "cast" to unsigned int (otherwise it will be negative numbers for bright colors)
          final int pixelR = inputArrayHW4[srcIdx] & 0xFF;
          final int pixelG = inputArrayHW4[srcIdx + 1] & 0xFF;
          final int pixelB = inputArrayHW4[srcIdx + 2] & 0xFF;
          mFloatBufferHW3[dstIdx] = inputScale * (float) pixelB - 1;
          mFloatBufferHW3[dstIdx + 1] = inputScale * (float) pixelG - 1;
          mFloatBufferHW3[dstIdx + 2] = inputScale * (float) pixelR - 1;
          srcIdx += 4;
          dstIdx += 3;
          sumG += pixelG;
      }
      // the buffer is black if on average on average Green < 13/255 (aka: 5%)
      mIsFloatBufferBlack = sumG < (area * 13);
      return mFloatBufferHW3;
  }

 MNETSSD_NUM_BOXES = mOutputs.get(MNETSSD_OUTPUT_LAYER).getSize();
  // convert tensors to boxes - Note: Optimized to read-all upfront
  mOutputs.get(MNETSSD_OUTPUT_LAYER).read(floatOutput, 0, MNETSSD_NUM_BOXES);
  //for black/white image
  int w = mScaledBitmap.getWidth();
  int h = mScaledBitmap.getHeight();
  int b = 0xFF;
  int out = 0xFF;
  for (int y = 0; y < h; y++) {
      for (int x = 0; x < w; x++) {
          b = b & mScaledBitmap.getPixel(x, y);
          for (int i = 1; i <= 3 && floatOutput[y * w + x] != 15; i++) {
              out = out << (8) | b;
          }
          mScaledBitmap.setPixel(x, y, floatOutput[y * w + x] != 15 ? out : mScaledBitmap.getPixel(x, y));
          out = 0xFF;
          b = 0xFF;
      }
  }
  mOutputBitmap = Bitmap.createScaledBitmap(mScaledBitmap, originalBitmapW,
          originalBitmapH, true);