diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java new file mode 100644 index 0000000..a9ab2d0 --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +/** Describing a frame info. */ +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + /** Builder of {@link FrameMetadata}. */ + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java new file mode 100644 index 0000000..a9ab2d0 --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +/** Describing a frame info. */ +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + /** Builder of {@link FrameMetadata}. */ + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index ecd425a..eb4308b 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -57,5 +57,5 @@ const val DEFAULT_SERVER_CONFIG = "defaultServerConfig" const val SERVER_BASE_URL = "http://111.198.10.15:22003" const val AI_SERVER_CONFIG = "aiServerConfig" - const val AI_BASE_URL = "http://192.168.14.127:5000" + const val AI_BASE_URL = "http://139.198.181.132:5000" } \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java new file mode 100644 index 0000000..a9ab2d0 --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +/** Describing a frame info. */ +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + /** Builder of {@link FrameMetadata}. */ + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index ecd425a..eb4308b 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -57,5 +57,5 @@ const val DEFAULT_SERVER_CONFIG = "defaultServerConfig" const val SERVER_BASE_URL = "http://111.198.10.15:22003" const val AI_SERVER_CONFIG = "aiServerConfig" - const val AI_BASE_URL = "http://192.168.14.127:5000" + const val AI_BASE_URL = "http://139.198.181.132:5000" } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt index 008e332..f52e4d1 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt @@ -30,6 +30,7 @@ import com.casic.br.app.databinding.ActivityStartCheckBinding import com.casic.br.app.extensions.initImmersionBar import com.casic.br.app.model.DictionaryModel +import com.casic.br.app.utils.BitmapUtils import com.casic.br.app.utils.LocaleConstant import com.casic.br.app.utils.LocationManager import com.casic.br.app.vm.ConfigViewModel @@ -128,19 +129,24 @@ imageFileViewModel = ViewModelProvider(this)[ImageFileViewModel::class.java] imageFileViewModel.sceneResult.observe(this) { - AlertControlDialog.Builder().setContext(this).setTitle("提示") - .setMessage("识别到目标场景，是否开始排查该场景的隐患？").setNegativeButton("稍后") - .setPositiveButton("好的").setOnDialogButtonClickListener(object : - AlertControlDialog.OnDialogButtonClickListener { - override fun onConfirmClick() { - binding.titleView.setTitle(it) - detectedScene = it - } + if (it == "没有检查到任何场景") { + isDetectingScene = false + } else { + AlertControlDialog.Builder().setContext(this).setTitle("提示") + .setMessage("识别到目标场景，是否开始排查该场景的隐患？") + .setNegativeButton("稍后") + .setPositiveButton("好的").setOnDialogButtonClickListener(object : + AlertControlDialog.OnDialogButtonClickListener { + override fun onConfirmClick() { + binding.titleView.setTitle(it) + detectedScene = it + } - override fun onCancelClick() { + override fun onCancelClick() { - } - }).build().show() + } + }).build().show() + } } imageFileViewModel.recognizeResult.observe(this) { @@ -158,7 +164,7 @@ * */ if (it.code == 200) { if (it.result.isEmpty()) { - binding.detectView.updateTargetPosition(null) + binding.detectView.clearTag() targetSet.clear() binding.tipsTagView.visibility = View.GONE } else { @@ -216,7 +222,8 @@ // ImageAnalysis imageAnalysis = ImageAnalysis.Builder().setTargetAspectRatio(screenAspectRatio) .setTargetRotation(Surface.ROTATION_0) - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() // Must unbind the use-cases before rebinding them cameraProvider.unbindAll() @@ -228,7 +235,7 @@ // Attach the viewfinder's surface provider to preview use case cameraPreViewBuilder.setSurfaceProvider(binding.cameraPreView.surfaceProvider) camera.cameraInfo.cameraState.observe(this) { - //开始预览之后才人脸检测 + //开始预览之后才检测 if (it.type == CameraState.Type.OPEN) { imageAnalysis.setAnalyzer(cameraExecutor, imageAnalyzer) } @@ -242,14 +249,14 @@ private val imageAnalyzer = ImageAnalysis.Analyzer { imageProxy -> if (imageProxy.format == ImageFormat.YUV_420_888) { executor.execute { - val image = imageProxy.image - image?.apply { - /** - * CameraX 原始预览Image数据（imageProxy.format == ImageFormat.YUV_420_888）转Bitmap - * */ - val yBuffer = this.planes[0].buffer - val uBuffer = this.planes[1].buffer - val vBuffer = this.planes[2].buffer + val bitmap = if (Build.BRAND == "Redmi" || Build.MANUFACTURER == "Xiaomi") { + BitmapUtils.getBitmap(imageProxy) ?: return@execute + } else { + val image = imageProxy.image ?: return@execute + + val yBuffer = image.planes[0].buffer + val uBuffer = image.planes[1].buffer + val vBuffer = image.planes[2].buffer val ySize = yBuffer.remaining() val uSize = uBuffer.remaining() @@ -261,30 +268,35 @@ uBuffer.get(nv21, ySize + vSize, uSize) val yuvImage = YuvImage( - nv21, ImageFormat.NV21, this.width, this.height, null + nv21, ImageFormat.NV21, image.width, image.height, null ) val out = ByteArrayOutputStream() yuvImage.compressToJpeg( Rect(0, 0, yuvImage.width, yuvImage.height), 100, out ) - val imageBytes = out.toByteArray() - val bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size) - val base64 = bitmap.rotateImage(90).encodeToBase64() - //先识别场景 - if (!isDetectingScene) { - isDetectingScene = true - imageFileViewModel.getScene(context, base64) - } - - //再识别隐患 - if (!isDetectingTarget && detectedScene.isNotBlank()) { - isDetectingTarget = true - imageFileViewModel.getRecognizeResult( - context, base64, detectedScene, inspectionId - ) - } + BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size).rotateImage(90) } + +// val path = "${createImageFileDir()}/${System.currentTimeMillis()}.png" +// Log.d(kTag, path) +// bitmap.saveImage(path) + val base64 = bitmap.encodeToBase64() + + //先识别场景 + if (!isDetectingScene) { + isDetectingScene = true + imageFileViewModel.getScene(context, base64) + } + + //再识别隐患 + if (!isDetectingTarget && detectedScene.isNotBlank()) { + isDetectingTarget = true + imageFileViewModel.getRecognizeResult( + context, base64, detectedScene, inspectionId + ) + } + //检测完之后close就会继续生成下一帧图片，否则就会被阻塞不会继续生成下一帧 imageProxy.close() } diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java new file mode 100644 index 0000000..a9ab2d0 --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +/** Describing a frame info. */ +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + /** Builder of {@link FrameMetadata}. */ + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index ecd425a..eb4308b 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -57,5 +57,5 @@ const val DEFAULT_SERVER_CONFIG = "defaultServerConfig" const val SERVER_BASE_URL = "http://111.198.10.15:22003" const val AI_SERVER_CONFIG = "aiServerConfig" - const val AI_BASE_URL = "http://192.168.14.127:5000" + const val AI_BASE_URL = "http://139.198.181.132:5000" } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt index 008e332..f52e4d1 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt @@ -30,6 +30,7 @@ import com.casic.br.app.databinding.ActivityStartCheckBinding import com.casic.br.app.extensions.initImmersionBar import com.casic.br.app.model.DictionaryModel +import com.casic.br.app.utils.BitmapUtils import com.casic.br.app.utils.LocaleConstant import com.casic.br.app.utils.LocationManager import com.casic.br.app.vm.ConfigViewModel @@ -128,19 +129,24 @@ imageFileViewModel = ViewModelProvider(this)[ImageFileViewModel::class.java] imageFileViewModel.sceneResult.observe(this) { - AlertControlDialog.Builder().setContext(this).setTitle("提示") - .setMessage("识别到目标场景，是否开始排查该场景的隐患？").setNegativeButton("稍后") - .setPositiveButton("好的").setOnDialogButtonClickListener(object : - AlertControlDialog.OnDialogButtonClickListener { - override fun onConfirmClick() { - binding.titleView.setTitle(it) - detectedScene = it - } + if (it == "没有检查到任何场景") { + isDetectingScene = false + } else { + AlertControlDialog.Builder().setContext(this).setTitle("提示") + .setMessage("识别到目标场景，是否开始排查该场景的隐患？") + .setNegativeButton("稍后") + .setPositiveButton("好的").setOnDialogButtonClickListener(object : + AlertControlDialog.OnDialogButtonClickListener { + override fun onConfirmClick() { + binding.titleView.setTitle(it) + detectedScene = it + } - override fun onCancelClick() { + override fun onCancelClick() { - } - }).build().show() + } + }).build().show() + } } imageFileViewModel.recognizeResult.observe(this) { @@ -158,7 +164,7 @@ * */ if (it.code == 200) { if (it.result.isEmpty()) { - binding.detectView.updateTargetPosition(null) + binding.detectView.clearTag() targetSet.clear() binding.tipsTagView.visibility = View.GONE } else { @@ -216,7 +222,8 @@ // ImageAnalysis imageAnalysis = ImageAnalysis.Builder().setTargetAspectRatio(screenAspectRatio) .setTargetRotation(Surface.ROTATION_0) - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() // Must unbind the use-cases before rebinding them cameraProvider.unbindAll() @@ -228,7 +235,7 @@ // Attach the viewfinder's surface provider to preview use case cameraPreViewBuilder.setSurfaceProvider(binding.cameraPreView.surfaceProvider) camera.cameraInfo.cameraState.observe(this) { - //开始预览之后才人脸检测 + //开始预览之后才检测 if (it.type == CameraState.Type.OPEN) { imageAnalysis.setAnalyzer(cameraExecutor, imageAnalyzer) } @@ -242,14 +249,14 @@ private val imageAnalyzer = ImageAnalysis.Analyzer { imageProxy -> if (imageProxy.format == ImageFormat.YUV_420_888) { executor.execute { - val image = imageProxy.image - image?.apply { - /** - * CameraX 原始预览Image数据（imageProxy.format == ImageFormat.YUV_420_888）转Bitmap - * */ - val yBuffer = this.planes[0].buffer - val uBuffer = this.planes[1].buffer - val vBuffer = this.planes[2].buffer + val bitmap = if (Build.BRAND == "Redmi" || Build.MANUFACTURER == "Xiaomi") { + BitmapUtils.getBitmap(imageProxy) ?: return@execute + } else { + val image = imageProxy.image ?: return@execute + + val yBuffer = image.planes[0].buffer + val uBuffer = image.planes[1].buffer + val vBuffer = image.planes[2].buffer val ySize = yBuffer.remaining() val uSize = uBuffer.remaining() @@ -261,30 +268,35 @@ uBuffer.get(nv21, ySize + vSize, uSize) val yuvImage = YuvImage( - nv21, ImageFormat.NV21, this.width, this.height, null + nv21, ImageFormat.NV21, image.width, image.height, null ) val out = ByteArrayOutputStream() yuvImage.compressToJpeg( Rect(0, 0, yuvImage.width, yuvImage.height), 100, out ) - val imageBytes = out.toByteArray() - val bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size) - val base64 = bitmap.rotateImage(90).encodeToBase64() - //先识别场景 - if (!isDetectingScene) { - isDetectingScene = true - imageFileViewModel.getScene(context, base64) - } - - //再识别隐患 - if (!isDetectingTarget && detectedScene.isNotBlank()) { - isDetectingTarget = true - imageFileViewModel.getRecognizeResult( - context, base64, detectedScene, inspectionId - ) - } + BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size).rotateImage(90) } + +// val path = "${createImageFileDir()}/${System.currentTimeMillis()}.png" +// Log.d(kTag, path) +// bitmap.saveImage(path) + val base64 = bitmap.encodeToBase64() + + //先识别场景 + if (!isDetectingScene) { + isDetectingScene = true + imageFileViewModel.getScene(context, base64) + } + + //再识别隐患 + if (!isDetectingTarget && detectedScene.isNotBlank()) { + isDetectingTarget = true + imageFileViewModel.getRecognizeResult( + context, base64, detectedScene, inspectionId + ) + } + //检测完之后close就会继续生成下一帧图片，否则就会被阻塞不会继续生成下一帧 imageProxy.close() } diff --git a/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt b/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt index 1529bc6..394bcdf 100644 --- a/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt +++ b/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt @@ -23,7 +23,6 @@ val resultModel = MutableLiveData() fun getScene(context: Context, base64: String) = launch({ - loadState.value = LoadState.Loading val response = RetrofitServiceManager.getScene(base64) when (response.getResponseCode()) { 200 -> { @@ -35,25 +34,17 @@ * */ val element = JsonParser.parseString(response) val jsonObject = element.asJsonObject - val scene = jsonObject.get("scene").asString - if (scene == "没有检查到任何场景") { - loadState.value = LoadState.Fail - scene.show(context) - } else { - loadState.value = LoadState.Success - sceneResult.value = scene - } + sceneResult.value = jsonObject.get("scene").asString } else -> { - loadState.value = LoadState.Fail val element = JsonParser.parseString(response) val jsonObject = element.asJsonObject jsonObject.get("result").asString.show(context) } } }, { - loadState.value = LoadState.Fail + it.printStackTrace() }) fun getRecognizeResult( diff --git a/app/build.gradle b/app/build.gradle index f3d6012..3d348bc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -106,11 +106,11 @@ implementation 'com.github.CarGuo.GSYVideoPlayer:GSYVideoPlayer:v8.4.0-release-jitpack' //CameraX def camerax_version = '1.2.3' - // CameraX Camera2 extensions + //CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" - // CameraX Lifecycle library + //CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" - // CameraX View class + //CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //PDF预览 implementation 'com.github.barteksc:android-pdf-viewer:2.8.2' diff --git a/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java new file mode 100644 index 0000000..4ddec0f --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/BitmapUtils.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.Image.Plane; +import android.net.Uri; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Utils functions for bitmap conversions. + */ +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + /** + * Converts NV21 format byte buffer to bitmap. + */ + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e(TAG, "Error: " + e.getMessage()); + } + return null; + } + + /** + * Converts a YUV_420_888 image from CameraX API to a bitmap. + */ + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + /** + * Rotates a bitmap if it is converted from a bytebuffer. + */ + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + // Rotate the image back to straight. + matrix.postRotate(rotationDegrees); + + // Mirror the image along the X or Y axis. + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + // Recycle the old bitmap if it has changed. + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each + // orientation. + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + // No transformations necessary in this case. + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + // We only support parsing EXIF orientation tag from local file on the device. + // See also: + // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + */ + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + */ + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + */ + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java new file mode 100644 index 0000000..a9ab2d0 --- /dev/null +++ b/app/src/main/java/com/casic/br/app/utils/FrameMetadata.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.casic.br.app.utils; + +/** Describing a frame info. */ +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + /** Builder of {@link FrameMetadata}. */ + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index ecd425a..eb4308b 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -57,5 +57,5 @@ const val DEFAULT_SERVER_CONFIG = "defaultServerConfig" const val SERVER_BASE_URL = "http://111.198.10.15:22003" const val AI_SERVER_CONFIG = "aiServerConfig" - const val AI_BASE_URL = "http://192.168.14.127:5000" + const val AI_BASE_URL = "http://139.198.181.132:5000" } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt index 008e332..f52e4d1 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckActivity.kt @@ -30,6 +30,7 @@ import com.casic.br.app.databinding.ActivityStartCheckBinding import com.casic.br.app.extensions.initImmersionBar import com.casic.br.app.model.DictionaryModel +import com.casic.br.app.utils.BitmapUtils import com.casic.br.app.utils.LocaleConstant import com.casic.br.app.utils.LocationManager import com.casic.br.app.vm.ConfigViewModel @@ -128,19 +129,24 @@ imageFileViewModel = ViewModelProvider(this)[ImageFileViewModel::class.java] imageFileViewModel.sceneResult.observe(this) { - AlertControlDialog.Builder().setContext(this).setTitle("提示") - .setMessage("识别到目标场景，是否开始排查该场景的隐患？").setNegativeButton("稍后") - .setPositiveButton("好的").setOnDialogButtonClickListener(object : - AlertControlDialog.OnDialogButtonClickListener { - override fun onConfirmClick() { - binding.titleView.setTitle(it) - detectedScene = it - } + if (it == "没有检查到任何场景") { + isDetectingScene = false + } else { + AlertControlDialog.Builder().setContext(this).setTitle("提示") + .setMessage("识别到目标场景，是否开始排查该场景的隐患？") + .setNegativeButton("稍后") + .setPositiveButton("好的").setOnDialogButtonClickListener(object : + AlertControlDialog.OnDialogButtonClickListener { + override fun onConfirmClick() { + binding.titleView.setTitle(it) + detectedScene = it + } - override fun onCancelClick() { + override fun onCancelClick() { - } - }).build().show() + } + }).build().show() + } } imageFileViewModel.recognizeResult.observe(this) { @@ -158,7 +164,7 @@ * */ if (it.code == 200) { if (it.result.isEmpty()) { - binding.detectView.updateTargetPosition(null) + binding.detectView.clearTag() targetSet.clear() binding.tipsTagView.visibility = View.GONE } else { @@ -216,7 +222,8 @@ // ImageAnalysis imageAnalysis = ImageAnalysis.Builder().setTargetAspectRatio(screenAspectRatio) .setTargetRotation(Surface.ROTATION_0) - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() // Must unbind the use-cases before rebinding them cameraProvider.unbindAll() @@ -228,7 +235,7 @@ // Attach the viewfinder's surface provider to preview use case cameraPreViewBuilder.setSurfaceProvider(binding.cameraPreView.surfaceProvider) camera.cameraInfo.cameraState.observe(this) { - //开始预览之后才人脸检测 + //开始预览之后才检测 if (it.type == CameraState.Type.OPEN) { imageAnalysis.setAnalyzer(cameraExecutor, imageAnalyzer) } @@ -242,14 +249,14 @@ private val imageAnalyzer = ImageAnalysis.Analyzer { imageProxy -> if (imageProxy.format == ImageFormat.YUV_420_888) { executor.execute { - val image = imageProxy.image - image?.apply { - /** - * CameraX 原始预览Image数据（imageProxy.format == ImageFormat.YUV_420_888）转Bitmap - * */ - val yBuffer = this.planes[0].buffer - val uBuffer = this.planes[1].buffer - val vBuffer = this.planes[2].buffer + val bitmap = if (Build.BRAND == "Redmi" || Build.MANUFACTURER == "Xiaomi") { + BitmapUtils.getBitmap(imageProxy) ?: return@execute + } else { + val image = imageProxy.image ?: return@execute + + val yBuffer = image.planes[0].buffer + val uBuffer = image.planes[1].buffer + val vBuffer = image.planes[2].buffer val ySize = yBuffer.remaining() val uSize = uBuffer.remaining() @@ -261,30 +268,35 @@ uBuffer.get(nv21, ySize + vSize, uSize) val yuvImage = YuvImage( - nv21, ImageFormat.NV21, this.width, this.height, null + nv21, ImageFormat.NV21, image.width, image.height, null ) val out = ByteArrayOutputStream() yuvImage.compressToJpeg( Rect(0, 0, yuvImage.width, yuvImage.height), 100, out ) - val imageBytes = out.toByteArray() - val bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size) - val base64 = bitmap.rotateImage(90).encodeToBase64() - //先识别场景 - if (!isDetectingScene) { - isDetectingScene = true - imageFileViewModel.getScene(context, base64) - } - - //再识别隐患 - if (!isDetectingTarget && detectedScene.isNotBlank()) { - isDetectingTarget = true - imageFileViewModel.getRecognizeResult( - context, base64, detectedScene, inspectionId - ) - } + BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size).rotateImage(90) } + +// val path = "${createImageFileDir()}/${System.currentTimeMillis()}.png" +// Log.d(kTag, path) +// bitmap.saveImage(path) + val base64 = bitmap.encodeToBase64() + + //先识别场景 + if (!isDetectingScene) { + isDetectingScene = true + imageFileViewModel.getScene(context, base64) + } + + //再识别隐患 + if (!isDetectingTarget && detectedScene.isNotBlank()) { + isDetectingTarget = true + imageFileViewModel.getRecognizeResult( + context, base64, detectedScene, inspectionId + ) + } + //检测完之后close就会继续生成下一帧图片，否则就会被阻塞不会继续生成下一帧 imageProxy.close() } diff --git a/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt b/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt index 1529bc6..394bcdf 100644 --- a/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt +++ b/app/src/main/java/com/casic/br/app/vm/ImageFileViewModel.kt @@ -23,7 +23,6 @@ val resultModel = MutableLiveData() fun getScene(context: Context, base64: String) = launch({ - loadState.value = LoadState.Loading val response = RetrofitServiceManager.getScene(base64) when (response.getResponseCode()) { 200 -> { @@ -35,25 +34,17 @@ * */ val element = JsonParser.parseString(response) val jsonObject = element.asJsonObject - val scene = jsonObject.get("scene").asString - if (scene == "没有检查到任何场景") { - loadState.value = LoadState.Fail - scene.show(context) - } else { - loadState.value = LoadState.Success - sceneResult.value = scene - } + sceneResult.value = jsonObject.get("scene").asString } else -> { - loadState.value = LoadState.Fail val element = JsonParser.parseString(response) val jsonObject = element.asJsonObject jsonObject.get("result").asString.show(context) } } }, { - loadState.value = LoadState.Fail + it.printStackTrace() }) fun getRecognizeResult( diff --git a/app/src/main/java/com/casic/br/app/widgets/AITargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/AITargetDetectView.kt index 8745fef..a6ffd26 100644 --- a/app/src/main/java/com/casic/br/app/widgets/AITargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/AITargetDetectView.kt @@ -17,13 +17,13 @@ class AITargetDetectView constructor(context: Context, attrs: AttributeSet) : View(context, attrs) { - private val kTag = "FaceDetectView" + private val kTag = "DetectView" private val textPaint by lazy { TextPaint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } private var viewWidth = 0 private var viewHeight = 0 - private var resultModels: MutableList? = null + private var resultModels: MutableList = ArrayList() init { textPaint.color = Color.RED @@ -41,32 +41,40 @@ viewHeight = context.getScreenHeight() - context.getStatusBarHeight() - 44.dp2px(context) } - fun updateTargetPosition(resultModels: MutableList?) { + fun updateTargetPosition(resultModels: MutableList) { this.resultModels = resultModels invalidate() } + fun clearTag() { + this.resultModels.clear() + invalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) - resultModels?.forEach { - //画文字 - val textLength = textPaint.measureText(it.label) - canvas.drawText( - it.label, - (it.box[0] * viewWidth + textLength / 2).toFloat(), - (it.box[1] * viewHeight - 10).toFloat(), - textPaint - ) + resultModels.forEach { +// {"box":[],"conf":0.7,"label":"电线杂乱"} + if (it.box.isNotEmpty()) { + //画文字 + val textLength = textPaint.measureText(it.label) + canvas.drawText( + it.label, + (it.box[0] * viewWidth + textLength / 2).toFloat(), + (it.box[1] * viewHeight - 10).toFloat(), + textPaint + ) - //画框 - val box = it.box - rect.set( - (box[0] * viewWidth).toInt(), - (box[1] * viewHeight).toInt(), - (box[2] * viewWidth).toInt(), - (box[3] * viewHeight).toInt() - ) - canvas.drawRect(rect, borderPaint) + //画框 + val box = it.box + rect.set( + (box[0] * viewWidth).toInt(), + (box[1] * viewHeight).toInt(), + (box[2] * viewWidth).toInt(), + (box[3] * viewHeight).toInt() + ) + canvas.drawRect(rect, borderPaint) + } } invalidate() }