diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/ndkcamera.cpp b/app/src/main/cpp/ndkcamera.cpp new file mode 100644 index 0000000..ed1e932 --- /dev/null +++ b/app/src/main/cpp/ndkcamera.cpp @@ -0,0 +1,699 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void *context, ACameraDevice *device) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void *context, ACameraDevice *device, int error) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void *context, AImageReader *reader) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage *image = nullptr; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = nullptr; + uint8_t *u_data = nullptr; + uint8_t *v_data = nullptr; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && + u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && + v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + auto *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *y_ptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + y_ptr[0] = y_data_ptr[0]; + y_ptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uv_ptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uv_ptr[0] = v_data_ptr[0]; + uv_ptr[1] = u_data_ptr[0]; + uv_ptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } + } + + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + ACameraCaptureFailure *failure) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, + failure); +} + +void onCaptureSequenceCompleted(void *context, ACameraCaptureSession *session, int sequenceId, + int64_t frameNumber) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", + session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void *context, ACameraCaptureSession *session, int sequenceId) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, + sequenceId); +} + +void onCaptureCompleted(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + const ACameraMetadata *result) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() { + camera_facing = 0; + camera_orientation = 0; + + camera_manager = nullptr; + camera_device = nullptr; + image_reader = nullptr; + image_reader_surface = nullptr; + image_reader_target = nullptr; + capture_request = nullptr; + capture_session_output_container = nullptr; + capture_session_output = nullptr; + capture_session = nullptr; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() { + close(); + + if (image_reader) { + AImageReader_delete(image_reader); + image_reader = nullptr; + } + + if (image_reader_surface) { + ANativeWindow_release(image_reader_surface); + image_reader_surface = nullptr; + } +} + +int NdkCamera::open(int _camera_facing) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList *camera_id_list = nullptr; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) { + const char *id = camera_id_list->cameraIds[i]; + ACameraMetadata *camera_metadata = nullptr; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t) e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int) e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), + camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, + &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, + capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, + &camera_capture_session_state_callbacks, + &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = nullptr; + camera_capture_session_capture_callbacks.onCaptureProgressed = nullptr; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = nullptr; + + ACameraCaptureSession_setRepeatingRequest(capture_session, + &camera_capture_session_capture_callbacks, 1, + &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = nullptr; + } + + if (camera_device) { + ACameraDevice_close(camera_device); + camera_device = nullptr; + } + + if (capture_session_output_container) { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = nullptr; + } + + if (capture_session_output) { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = nullptr; + } + + if (capture_request) { + ACaptureRequest_free(capture_request); + capture_request = nullptr; + } + + if (image_reader_target) { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = nullptr; + } + + if (camera_manager) { + ACameraManager_delete(camera_manager); + camera_manager = nullptr; + } +} + +void NdkCamera::on_image(const cv::Mat &rgb) const { +} + +void NdkCamera::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, + rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() { + sensor_manager = nullptr; + sensor_event_queue = nullptr; + accelerometer_sensor = nullptr; + win = nullptr; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, + ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() { + if (accelerometer_sensor) { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = nullptr; + } + + if (sensor_event_queue) { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = nullptr; + } + + if (win) { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow *_win) { + if (win) { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat &rgb) const { +} + +void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare( + ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, nullptr, nullptr, nullptr); + if (id == NDKCAMERAWINDOW_ID) { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) { + if (win_w * nv21_height > win_h * nv21_width) { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) { + if (win_w * nv21_width > win_h * nv21_height) { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) { + if (camera_orientation == 0 && accelerometer_orientation == 0) { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) { + rotate_type = 2; + } + } else { + if (final_orientation == 0) { + rotate_type = 1; + } + if (final_orientation == 90) { + rotate_type = 6; + } + if (final_orientation == 180) { + rotate_type = 3; + } + if (final_orientation == 270) { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char *srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char *dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, + rotate_type); + + const unsigned char *srcUV = + nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char *dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, + roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, + render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, + AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || + buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) { + for (int y = 0; y < render_h; y++) { + const auto *ptr = rgb_render.ptr(y); + unsigned char *outptr = (unsigned char *) buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/ndkcamera.cpp b/app/src/main/cpp/ndkcamera.cpp new file mode 100644 index 0000000..ed1e932 --- /dev/null +++ b/app/src/main/cpp/ndkcamera.cpp @@ -0,0 +1,699 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void *context, ACameraDevice *device) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void *context, ACameraDevice *device, int error) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void *context, AImageReader *reader) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage *image = nullptr; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = nullptr; + uint8_t *u_data = nullptr; + uint8_t *v_data = nullptr; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && + u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && + v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + auto *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *y_ptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + y_ptr[0] = y_data_ptr[0]; + y_ptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uv_ptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uv_ptr[0] = v_data_ptr[0]; + uv_ptr[1] = u_data_ptr[0]; + uv_ptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } + } + + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + ACameraCaptureFailure *failure) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, + failure); +} + +void onCaptureSequenceCompleted(void *context, ACameraCaptureSession *session, int sequenceId, + int64_t frameNumber) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", + session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void *context, ACameraCaptureSession *session, int sequenceId) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, + sequenceId); +} + +void onCaptureCompleted(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + const ACameraMetadata *result) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() { + camera_facing = 0; + camera_orientation = 0; + + camera_manager = nullptr; + camera_device = nullptr; + image_reader = nullptr; + image_reader_surface = nullptr; + image_reader_target = nullptr; + capture_request = nullptr; + capture_session_output_container = nullptr; + capture_session_output = nullptr; + capture_session = nullptr; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() { + close(); + + if (image_reader) { + AImageReader_delete(image_reader); + image_reader = nullptr; + } + + if (image_reader_surface) { + ANativeWindow_release(image_reader_surface); + image_reader_surface = nullptr; + } +} + +int NdkCamera::open(int _camera_facing) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList *camera_id_list = nullptr; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) { + const char *id = camera_id_list->cameraIds[i]; + ACameraMetadata *camera_metadata = nullptr; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t) e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int) e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), + camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, + &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, + capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, + &camera_capture_session_state_callbacks, + &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = nullptr; + camera_capture_session_capture_callbacks.onCaptureProgressed = nullptr; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = nullptr; + + ACameraCaptureSession_setRepeatingRequest(capture_session, + &camera_capture_session_capture_callbacks, 1, + &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = nullptr; + } + + if (camera_device) { + ACameraDevice_close(camera_device); + camera_device = nullptr; + } + + if (capture_session_output_container) { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = nullptr; + } + + if (capture_session_output) { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = nullptr; + } + + if (capture_request) { + ACaptureRequest_free(capture_request); + capture_request = nullptr; + } + + if (image_reader_target) { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = nullptr; + } + + if (camera_manager) { + ACameraManager_delete(camera_manager); + camera_manager = nullptr; + } +} + +void NdkCamera::on_image(const cv::Mat &rgb) const { +} + +void NdkCamera::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, + rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() { + sensor_manager = nullptr; + sensor_event_queue = nullptr; + accelerometer_sensor = nullptr; + win = nullptr; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, + ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() { + if (accelerometer_sensor) { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = nullptr; + } + + if (sensor_event_queue) { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = nullptr; + } + + if (win) { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow *_win) { + if (win) { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat &rgb) const { +} + +void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare( + ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, nullptr, nullptr, nullptr); + if (id == NDKCAMERAWINDOW_ID) { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) { + if (win_w * nv21_height > win_h * nv21_width) { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) { + if (win_w * nv21_width > win_h * nv21_height) { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) { + if (camera_orientation == 0 && accelerometer_orientation == 0) { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) { + rotate_type = 2; + } + } else { + if (final_orientation == 0) { + rotate_type = 1; + } + if (final_orientation == 90) { + rotate_type = 6; + } + if (final_orientation == 180) { + rotate_type = 3; + } + if (final_orientation == 270) { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char *srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char *dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, + rotate_type); + + const unsigned char *srcUV = + nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char *dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, + roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, + render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, + AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || + buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) { + for (int y = 0; y < render_h; y++) { + const auto *ptr = rgb_render.ptr(y); + unsigned char *outptr = (unsigned char *) buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/app/src/main/cpp/ndkcamera.h b/app/src/main/cpp/ndkcamera.h new file mode 100644 index 0000000..92abbaf --- /dev/null +++ b/app/src/main/cpp/ndkcamera.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef NDKCAMERA_H +#define NDKCAMERA_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +class NdkCamera { +public: + NdkCamera(); + + virtual ~NdkCamera(); + + // facing 0=front 1=back + int open(int camera_facing = 0); + + void close(); + + virtual void on_image(const cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + int camera_facing; + int camera_orientation; + +private: + ACameraManager *camera_manager; + ACameraDevice *camera_device; + AImageReader *image_reader; + ANativeWindow *image_reader_surface; + ACameraOutputTarget *image_reader_target; + ACaptureRequest *capture_request; + ACaptureSessionOutputContainer *capture_session_output_container; + ACaptureSessionOutput *capture_session_output; + ACameraCaptureSession *capture_session; +}; + +class NdkCameraWindow : public NdkCamera { +public: + NdkCameraWindow(); + + virtual ~NdkCameraWindow(); + + void set_window(ANativeWindow *win); + + virtual void on_image_render(cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + mutable int accelerometer_orientation; + +private: + ASensorManager *sensor_manager; + mutable ASensorEventQueue *sensor_event_queue; + const ASensor *accelerometer_sensor; + ANativeWindow *win; +}; + +#endif // NDKCAMERA_H diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/ndkcamera.cpp b/app/src/main/cpp/ndkcamera.cpp new file mode 100644 index 0000000..ed1e932 --- /dev/null +++ b/app/src/main/cpp/ndkcamera.cpp @@ -0,0 +1,699 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void *context, ACameraDevice *device) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void *context, ACameraDevice *device, int error) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void *context, AImageReader *reader) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage *image = nullptr; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = nullptr; + uint8_t *u_data = nullptr; + uint8_t *v_data = nullptr; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && + u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && + v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + auto *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *y_ptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + y_ptr[0] = y_data_ptr[0]; + y_ptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uv_ptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uv_ptr[0] = v_data_ptr[0]; + uv_ptr[1] = u_data_ptr[0]; + uv_ptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } + } + + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + ACameraCaptureFailure *failure) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, + failure); +} + +void onCaptureSequenceCompleted(void *context, ACameraCaptureSession *session, int sequenceId, + int64_t frameNumber) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", + session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void *context, ACameraCaptureSession *session, int sequenceId) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, + sequenceId); +} + +void onCaptureCompleted(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + const ACameraMetadata *result) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() { + camera_facing = 0; + camera_orientation = 0; + + camera_manager = nullptr; + camera_device = nullptr; + image_reader = nullptr; + image_reader_surface = nullptr; + image_reader_target = nullptr; + capture_request = nullptr; + capture_session_output_container = nullptr; + capture_session_output = nullptr; + capture_session = nullptr; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() { + close(); + + if (image_reader) { + AImageReader_delete(image_reader); + image_reader = nullptr; + } + + if (image_reader_surface) { + ANativeWindow_release(image_reader_surface); + image_reader_surface = nullptr; + } +} + +int NdkCamera::open(int _camera_facing) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList *camera_id_list = nullptr; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) { + const char *id = camera_id_list->cameraIds[i]; + ACameraMetadata *camera_metadata = nullptr; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t) e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int) e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), + camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, + &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, + capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, + &camera_capture_session_state_callbacks, + &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = nullptr; + camera_capture_session_capture_callbacks.onCaptureProgressed = nullptr; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = nullptr; + + ACameraCaptureSession_setRepeatingRequest(capture_session, + &camera_capture_session_capture_callbacks, 1, + &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = nullptr; + } + + if (camera_device) { + ACameraDevice_close(camera_device); + camera_device = nullptr; + } + + if (capture_session_output_container) { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = nullptr; + } + + if (capture_session_output) { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = nullptr; + } + + if (capture_request) { + ACaptureRequest_free(capture_request); + capture_request = nullptr; + } + + if (image_reader_target) { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = nullptr; + } + + if (camera_manager) { + ACameraManager_delete(camera_manager); + camera_manager = nullptr; + } +} + +void NdkCamera::on_image(const cv::Mat &rgb) const { +} + +void NdkCamera::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, + rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() { + sensor_manager = nullptr; + sensor_event_queue = nullptr; + accelerometer_sensor = nullptr; + win = nullptr; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, + ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() { + if (accelerometer_sensor) { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = nullptr; + } + + if (sensor_event_queue) { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = nullptr; + } + + if (win) { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow *_win) { + if (win) { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat &rgb) const { +} + +void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare( + ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, nullptr, nullptr, nullptr); + if (id == NDKCAMERAWINDOW_ID) { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) { + if (win_w * nv21_height > win_h * nv21_width) { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) { + if (win_w * nv21_width > win_h * nv21_height) { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) { + if (camera_orientation == 0 && accelerometer_orientation == 0) { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) { + rotate_type = 2; + } + } else { + if (final_orientation == 0) { + rotate_type = 1; + } + if (final_orientation == 90) { + rotate_type = 6; + } + if (final_orientation == 180) { + rotate_type = 3; + } + if (final_orientation == 270) { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char *srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char *dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, + rotate_type); + + const unsigned char *srcUV = + nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char *dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, + roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, + render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, + AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || + buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) { + for (int y = 0; y < render_h; y++) { + const auto *ptr = rgb_render.ptr(y); + unsigned char *outptr = (unsigned char *) buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/app/src/main/cpp/ndkcamera.h b/app/src/main/cpp/ndkcamera.h new file mode 100644 index 0000000..92abbaf --- /dev/null +++ b/app/src/main/cpp/ndkcamera.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef NDKCAMERA_H +#define NDKCAMERA_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +class NdkCamera { +public: + NdkCamera(); + + virtual ~NdkCamera(); + + // facing 0=front 1=back + int open(int camera_facing = 0); + + void close(); + + virtual void on_image(const cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + int camera_facing; + int camera_orientation; + +private: + ACameraManager *camera_manager; + ACameraDevice *camera_device; + AImageReader *image_reader; + ANativeWindow *image_reader_surface; + ACameraOutputTarget *image_reader_target; + ACaptureRequest *capture_request; + ACaptureSessionOutputContainer *capture_session_output_container; + ACaptureSessionOutput *capture_session_output; + ACameraCaptureSession *capture_session; +}; + +class NdkCameraWindow : public NdkCamera { +public: + NdkCameraWindow(); + + virtual ~NdkCameraWindow(); + + void set_window(ANativeWindow *win); + + virtual void on_image_render(cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + mutable int accelerometer_orientation; + +private: + ASensorManager *sensor_manager; + mutable ASensorEventQueue *sensor_event_queue; + const ASensor *accelerometer_sensor; + ANativeWindow *win; +}; + +#endif // NDKCAMERA_H diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp new file mode 100644 index 0000000..4195087 --- /dev/null +++ b/app/src/main/cpp/yolo.cpp @@ -0,0 +1,853 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "yolo.h" + +#include +#include + +#include + +#include "cpu.h" + +static float fast_exp(float x) { + union { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; +} + +static float sigmoid(float x) { + return 1.0f / (1.0f + fast_exp(-x)); +} + +static float intersection_area(const Object &a, const Object &b) { + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); +} + +static void qsort_descent_inplace(std::vector &face_objects, int left, int right) { + int i = left; + int j = right; + float p = face_objects[(left + right) / 2].prob; + + while (i <= j) { + while (face_objects[i].prob > p) + i++; + + while (face_objects[j].prob < p) + j--; + + if (i <= j) { + // swap + std::swap(face_objects[i], face_objects[j]); + + i++; + j--; + } + } + + // #pragma omp parallel sections + { + // #pragma omp section + { + if (left < j) qsort_descent_inplace(face_objects, left, j); + } + // #pragma omp section + { + if (i < right) qsort_descent_inplace(face_objects, i, right); + } + } +} + +static void qsort_descent_inplace(std::vector &face_objects) { + if (face_objects.empty()) + return; + + qsort_descent_inplace(face_objects, 0, face_objects.size() - 1); +} + +static void nms_sorted_bboxes(const std::vector &face_objects, std::vector &picked, + float nms_threshold) { + picked.clear(); + + const int n = face_objects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) { + areas[i] = face_objects[i].rect.width * face_objects[i].rect.height; + } + + for (int i = 0; i < n; i++) { + const Object &a = face_objects[i]; + + int keep = 1; + for (int j = 0; j < (int) picked.size(); j++) { + const Object &b = face_objects[picked[j]]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } +} + +static void +generate_grids_and_stride(const int target_w, const int target_h, std::vector &strides, + std::vector &grid_strides) { + for (int stride: strides) { + int num_grid_w = target_w / stride; + int num_grid_h = target_h / stride; + for (int g1 = 0; g1 < num_grid_h; g1++) { + for (int g0 = 0; g0 < num_grid_w; g0++) { + GridAndStride gs; + gs.grid0 = g0; + gs.grid1 = g1; + gs.stride = stride; + grid_strides.push_back(gs); + } + } + } +} + +static void generate_proposals(std::vector grid_strides, const ncnn::Mat &pred, + float prob_threshold, std::vector &objects, int num_class) { + const int num_points = grid_strides.size(); + const int reg_max_1 = 16; + + for (int i = 0; i < num_points; i++) { + const float *scores = pred.row(i) + 4 * reg_max_1; + + // find label with max score + int label = -1; + float score = -FLT_MAX; + for (int k = 0; k < num_class; k++) { + float confidence = scores[k]; + if (confidence > score) { + label = k; + score = confidence; + } + } + float box_prob = sigmoid(score); + if (box_prob >= prob_threshold) { + ncnn::Mat bbox_pred(reg_max_1, 4, (void *) pred.row(i)); + { + ncnn::Layer *softmax = ncnn::create_layer("Softmax"); + + ncnn::ParamDict pd; + pd.set(0, 1); // axis + pd.set(1, 1); + softmax->load_param(pd); + + ncnn::Option opt; + opt.num_threads = 1; + opt.use_packing_layout = false; + + softmax->create_pipeline(opt); + + softmax->forward_inplace(bbox_pred, opt); + + softmax->destroy_pipeline(opt); + + delete softmax; + } + + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) { + float dis = 0.f; + const float *dis_after_sm = bbox_pred.row(k); + for (int l = 0; l < reg_max_1; l++) { + dis += l * dis_after_sm[l]; + } + + pred_ltrb[k] = dis * grid_strides[i].stride; + } + + float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; + float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = label; + obj.prob = box_prob; + + objects.push_back(obj); + } + } +} + +/***模型分割*************/ +static void matmul(const std::vector &bottom_blobs, ncnn::Mat &top_blob) { + ncnn::Option opt; + opt.num_threads = 2; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("MatMul"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 0);// axis + + op->load_param(pd); + + op->create_pipeline(opt); + std::vector top_blobs(1); + op->forward(bottom_blobs, top_blobs, opt); + top_blob = top_blobs[0]; + + op->destroy_pipeline(opt); + + delete op; +} + +static void sigmoid(ncnn::Mat &bottom) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Sigmoid"); + + op->create_pipeline(opt); + + // forward + + op->forward_inplace(bottom, opt); + op->destroy_pipeline(opt); + + delete op; +} + +static void reshape(const ncnn::Mat &in, ncnn::Mat &out, int c, int h, int w, int d) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Reshape"); + + // set param + ncnn::ParamDict pd; + + pd.set(0, w);// start + pd.set(1, h);// end + if (d > 0) + pd.set(11, d);//axes + pd.set(2, c);//axes + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void slice(const ncnn::Mat &in, ncnn::Mat &out, int start, int end, int axis) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Crop"); + + // set param + ncnn::ParamDict pd; + + ncnn::Mat axes = ncnn::Mat(1); + axes.fill(axis); + ncnn::Mat ends = ncnn::Mat(1); + ends.fill(end); + ncnn::Mat starts = ncnn::Mat(1); + starts.fill(start); + pd.set(9, starts);// start + pd.set(10, ends);// end + pd.set(11, axes);//axes + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void interp(const ncnn::Mat &in, const float &scale, const int &out_w, const int &out_h, + ncnn::Mat &out) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Interp"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 2);// resize_type + pd.set(1, scale);// height_scale + pd.set(2, scale);// width_scale + pd.set(3, out_h);// height + pd.set(4, out_w);// width + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void decode_mask(const ncnn::Mat &mask_feat, const int &img_w, const int &img_h, + const ncnn::Mat &mask_proto, const ncnn::Mat &in_pad, const int &wpad, + const int &hpad, ncnn::Mat &mask_pred_result) { + ncnn::Mat masks; + matmul(std::vector{mask_feat, mask_proto}, masks); + sigmoid(masks); + reshape(masks, masks, masks.h, in_pad.h / 4, in_pad.w / 4, 0); + slice(masks, mask_pred_result, (wpad / 2) / 4, (in_pad.w - wpad / 2) / 4, 2); + slice(mask_pred_result, mask_pred_result, (hpad / 2) / 4, (in_pad.h - hpad / 2) / 4, 1); + interp(mask_pred_result, 4.0, img_w, img_h, mask_pred_result); +} + +/***模型分割*************/ + +Yolo::Yolo() { + blob_pool_allocator.set_size_compare_ratio(0.f); + workspace_pool_allocator.set_size_compare_ratio(0.f); +} + +int +Yolo::load(const char *model_type, int target_size, const float *mean_values, + const float *norm_values, bool use_gpu) { + +} + +int +Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, + const float *_norm_values, bool use_gpu) { + yolo.clear(); + blob_pool_allocator.clear(); + workspace_pool_allocator.clear(); + + ncnn::set_cpu_powersave(2); + ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); + + yolo.opt = ncnn::Option(); + +#if NCNN_VULKAN + yolo.opt.use_vulkan_compute = use_gpu; +#endif + + yolo.opt.num_threads = ncnn::get_big_cpu_count(); + yolo.opt.blob_allocator = &blob_pool_allocator; + yolo.opt.workspace_allocator = &workspace_pool_allocator; + + char param_path[256]; + char model_path[256]; + //拼接模型名（路径） + sprintf(param_path, "%s.param", model_type); + sprintf(model_path, "%s.bin", model_type); + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); + + yolo.load_param(mgr, param_path); + yolo.load_model(mgr, model_path); + + target_size = _target_size; + mean_values[0] = _mean_values[0]; + mean_values[1] = _mean_values[1]; + mean_values[2] = _mean_values[2]; + norm_values[0] = _norm_values[0]; + norm_values[1] = _norm_values[1]; + norm_values[2] = _norm_values[2]; + + return 0; +} + +void Yolo::initNativeCallback(JavaVM *vm, jobject input, jlong nativeObjAddr, jobject pJobject) { + javaVM = vm; + + /** + * JNIEnv不支持跨线程调用 + * */ + JNIEnv *env; + vm->AttachCurrentThread(&env, nullptr); + //此时input转为output + j_output = env->NewGlobalRef(input); + + j_mat_addr = nativeObjAddr; + + j_callback = env->NewGlobalRef(pJobject); +} + +int Yolo::classify(const cv::Mat &rgb) { + if (state == 0) { + static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; + + int width = rgb.cols; + int height = rgb.rows; + + //把opencv Mat转为 ncnn Mat + ncnn::Mat in = ncnn::Mat::from_pixels(rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height); + + std::vector cls_scores; + { + in.substract_mean_normalize(mean_values, scale_values); + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in); + + ncnn::Mat out; + ex.extract("output", out); + + int output_size = out.w; + float float_buffer[output_size]; + for (int j = 0; j < out.w; j++) { + float_buffer[j] = out[j]; + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID(callback_clazz, "onClassify", "([F)V"); + + jfloatArray j_output_Data = env->NewFloatArray(output_size); + env->SetFloatArrayRegion(j_output_Data, 0, output_size, float_buffer); + + env->CallVoidMethod(j_callback, j_method_id, j_output_Data); + } + } + return 0; +} + +int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 1) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_BGR2RGB, width, + height, w, h); + + // pad to target_size rectangle + int wpad = (w + 31) / 32 * 32 - w; + int hpad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, + ncnn::BORDER_CONSTANT, 0.f); + + const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; + in_pad.substract_mean_normalize(0, norm_vals); + + + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (wpad / 2)) / scale; + float y0 = (objects[i].rect.y - (hpad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); + + for (auto item: objects) { + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + + /** + * Mat数据。 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 2) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale = 1.f; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize( + rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height, w, h + ); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border( + in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, + ncnn::BORDER_CONSTANT, 0.f + ); + + in_pad.substract_mean_normalize(0, norm_values); + + ncnn::Extractor ex = yolo.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, prob_threshold, proposals, 43); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + /** + * I: 整数类型（int） + * J: 长整数类型（long） + * D: 双精度浮点数类型（double） + * F: 单精度浮点数类型（float） + * Z: 布尔类型（boolean） + * C: 字符类型（char） + * B: 字节类型（byte） + * S: 短整数类型（short） + *
-----------------------------------------------
+ * Ljava/lang/Object;: 表示 Object 类型的引用 + * Ljava/lang/String;: 表示 String 类型的引用 + * L包名/类名;: 表示特定包名和类名的引用 + *
-----------------------------------------------
+ * 例如： + * int add(int a, int b): (II)I + * + * String concat(String str1, String str2): (Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String; + *
-----------------------------------------------
+ * [Ljava/lang/String;: 表示 String 类型的一维数组 + * */ + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Lcom/casic/br/ar/app/external/YoloResult;)V" + ); + + for (int i = 0; i < count; i++) { + auto item = objects[i]; + + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //回调 + env->CallVoidMethod(j_callback, j_method_id, j_output); + } + + /** + * Mat数据。 + *
-----------------------------------------------
+ * 通过内存地址赋值。Java层传入Mat对象内存地址，再通过C++给此地址赋值，Java即可得到内存地址的Mat矩阵数据 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::draw(cv::Mat &rgb, const std::vector &objects) { + static const char *class_names[] = { + "tripod", "tee", "person", + "shut-off valve", "hazard signs", "pressure tester", + "pressure gauge", "reflective clothing", "respirator masks", + "throat foil", "round-headed water gun", "safety signs", + "helmet", "security identification", "safety ropes", + "intercom", "pointed water gun", "switch", + "alarm device", "joint", "construction street signs", + "gas detectors", "hoses", "hose_rectangle", + "flow-meter", "fire hydrant box", "fire extinguisher", + "lighting equipment", "flame-out protection", "exposed wires", + "circuit diagram", "cordon", "regulator", + "length adjuster", "stickers", "across wires", + "road cones", "hose", "filter", + "distribution box", "long-shank valves", "valve", "ducts" + }; + + static const unsigned char colors[19][3] = { + {54, 67, 244}, + {99, 30, 233}, + {176, 39, 156}, + {183, 58, 103}, + {181, 81, 63}, + {243, 150, 33}, + {244, 169, 3}, + {212, 188, 0}, + {136, 150, 0}, + {80, 175, 76}, + {74, 195, 139}, + {57, 220, 205}, + {59, 235, 255}, + {7, 193, 255}, + {0, 152, 255}, + {34, 87, 255}, + {72, 85, 121}, + {158, 158, 158}, + {139, 125, 96} + }; + + int color_index = 0; + + for (const auto &obj: objects) { + const unsigned char *color = colors[color_index % 19]; + + color_index++; + + cv::Scalar cc(color[0], color[1], color[2]); + + cv::rectangle(rgb, obj.rect, cc, 2); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, + &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > rgb.cols) + x = rgb.cols - label_size.width; + + cv::Size size = cv::Size(label_size.width, label_size.height + baseLine); + cv::Rect rc = cv::Rect(cv::Point(x, y), size); + cv::rectangle(rgb, rc, cc, -1); + + cv::Scalar text_scalar = (color[0] + color[1] + color[2] >= 381) + ? cv::Scalar(0, 0, 0) + : cv::Scalar(255, 255, 255); + + + cv::putText(rgb, text, + cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, + 0.5, + text_scalar, 1 + ); + } + return 0; +} \ No newline at end of file diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/ndkcamera.cpp b/app/src/main/cpp/ndkcamera.cpp new file mode 100644 index 0000000..ed1e932 --- /dev/null +++ b/app/src/main/cpp/ndkcamera.cpp @@ -0,0 +1,699 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void *context, ACameraDevice *device) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void *context, ACameraDevice *device, int error) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void *context, AImageReader *reader) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage *image = nullptr; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = nullptr; + uint8_t *u_data = nullptr; + uint8_t *v_data = nullptr; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && + u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && + v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + auto *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *y_ptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + y_ptr[0] = y_data_ptr[0]; + y_ptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uv_ptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uv_ptr[0] = v_data_ptr[0]; + uv_ptr[1] = u_data_ptr[0]; + uv_ptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } + } + + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + ACameraCaptureFailure *failure) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, + failure); +} + +void onCaptureSequenceCompleted(void *context, ACameraCaptureSession *session, int sequenceId, + int64_t frameNumber) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", + session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void *context, ACameraCaptureSession *session, int sequenceId) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, + sequenceId); +} + +void onCaptureCompleted(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + const ACameraMetadata *result) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() { + camera_facing = 0; + camera_orientation = 0; + + camera_manager = nullptr; + camera_device = nullptr; + image_reader = nullptr; + image_reader_surface = nullptr; + image_reader_target = nullptr; + capture_request = nullptr; + capture_session_output_container = nullptr; + capture_session_output = nullptr; + capture_session = nullptr; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() { + close(); + + if (image_reader) { + AImageReader_delete(image_reader); + image_reader = nullptr; + } + + if (image_reader_surface) { + ANativeWindow_release(image_reader_surface); + image_reader_surface = nullptr; + } +} + +int NdkCamera::open(int _camera_facing) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList *camera_id_list = nullptr; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) { + const char *id = camera_id_list->cameraIds[i]; + ACameraMetadata *camera_metadata = nullptr; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t) e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int) e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), + camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, + &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, + capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, + &camera_capture_session_state_callbacks, + &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = nullptr; + camera_capture_session_capture_callbacks.onCaptureProgressed = nullptr; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = nullptr; + + ACameraCaptureSession_setRepeatingRequest(capture_session, + &camera_capture_session_capture_callbacks, 1, + &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = nullptr; + } + + if (camera_device) { + ACameraDevice_close(camera_device); + camera_device = nullptr; + } + + if (capture_session_output_container) { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = nullptr; + } + + if (capture_session_output) { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = nullptr; + } + + if (capture_request) { + ACaptureRequest_free(capture_request); + capture_request = nullptr; + } + + if (image_reader_target) { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = nullptr; + } + + if (camera_manager) { + ACameraManager_delete(camera_manager); + camera_manager = nullptr; + } +} + +void NdkCamera::on_image(const cv::Mat &rgb) const { +} + +void NdkCamera::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, + rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() { + sensor_manager = nullptr; + sensor_event_queue = nullptr; + accelerometer_sensor = nullptr; + win = nullptr; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, + ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() { + if (accelerometer_sensor) { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = nullptr; + } + + if (sensor_event_queue) { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = nullptr; + } + + if (win) { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow *_win) { + if (win) { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat &rgb) const { +} + +void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare( + ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, nullptr, nullptr, nullptr); + if (id == NDKCAMERAWINDOW_ID) { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) { + if (win_w * nv21_height > win_h * nv21_width) { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) { + if (win_w * nv21_width > win_h * nv21_height) { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) { + if (camera_orientation == 0 && accelerometer_orientation == 0) { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) { + rotate_type = 2; + } + } else { + if (final_orientation == 0) { + rotate_type = 1; + } + if (final_orientation == 90) { + rotate_type = 6; + } + if (final_orientation == 180) { + rotate_type = 3; + } + if (final_orientation == 270) { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char *srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char *dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, + rotate_type); + + const unsigned char *srcUV = + nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char *dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, + roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, + render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, + AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || + buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) { + for (int y = 0; y < render_h; y++) { + const auto *ptr = rgb_render.ptr(y); + unsigned char *outptr = (unsigned char *) buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/app/src/main/cpp/ndkcamera.h b/app/src/main/cpp/ndkcamera.h new file mode 100644 index 0000000..92abbaf --- /dev/null +++ b/app/src/main/cpp/ndkcamera.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef NDKCAMERA_H +#define NDKCAMERA_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +class NdkCamera { +public: + NdkCamera(); + + virtual ~NdkCamera(); + + // facing 0=front 1=back + int open(int camera_facing = 0); + + void close(); + + virtual void on_image(const cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + int camera_facing; + int camera_orientation; + +private: + ACameraManager *camera_manager; + ACameraDevice *camera_device; + AImageReader *image_reader; + ANativeWindow *image_reader_surface; + ACameraOutputTarget *image_reader_target; + ACaptureRequest *capture_request; + ACaptureSessionOutputContainer *capture_session_output_container; + ACaptureSessionOutput *capture_session_output; + ACameraCaptureSession *capture_session; +}; + +class NdkCameraWindow : public NdkCamera { +public: + NdkCameraWindow(); + + virtual ~NdkCameraWindow(); + + void set_window(ANativeWindow *win); + + virtual void on_image_render(cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + mutable int accelerometer_orientation; + +private: + ASensorManager *sensor_manager; + mutable ASensorEventQueue *sensor_event_queue; + const ASensor *accelerometer_sensor; + ANativeWindow *win; +}; + +#endif // NDKCAMERA_H diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp new file mode 100644 index 0000000..4195087 --- /dev/null +++ b/app/src/main/cpp/yolo.cpp @@ -0,0 +1,853 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "yolo.h" + +#include +#include + +#include + +#include "cpu.h" + +static float fast_exp(float x) { + union { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; +} + +static float sigmoid(float x) { + return 1.0f / (1.0f + fast_exp(-x)); +} + +static float intersection_area(const Object &a, const Object &b) { + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); +} + +static void qsort_descent_inplace(std::vector &face_objects, int left, int right) { + int i = left; + int j = right; + float p = face_objects[(left + right) / 2].prob; + + while (i <= j) { + while (face_objects[i].prob > p) + i++; + + while (face_objects[j].prob < p) + j--; + + if (i <= j) { + // swap + std::swap(face_objects[i], face_objects[j]); + + i++; + j--; + } + } + + // #pragma omp parallel sections + { + // #pragma omp section + { + if (left < j) qsort_descent_inplace(face_objects, left, j); + } + // #pragma omp section + { + if (i < right) qsort_descent_inplace(face_objects, i, right); + } + } +} + +static void qsort_descent_inplace(std::vector &face_objects) { + if (face_objects.empty()) + return; + + qsort_descent_inplace(face_objects, 0, face_objects.size() - 1); +} + +static void nms_sorted_bboxes(const std::vector &face_objects, std::vector &picked, + float nms_threshold) { + picked.clear(); + + const int n = face_objects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) { + areas[i] = face_objects[i].rect.width * face_objects[i].rect.height; + } + + for (int i = 0; i < n; i++) { + const Object &a = face_objects[i]; + + int keep = 1; + for (int j = 0; j < (int) picked.size(); j++) { + const Object &b = face_objects[picked[j]]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } +} + +static void +generate_grids_and_stride(const int target_w, const int target_h, std::vector &strides, + std::vector &grid_strides) { + for (int stride: strides) { + int num_grid_w = target_w / stride; + int num_grid_h = target_h / stride; + for (int g1 = 0; g1 < num_grid_h; g1++) { + for (int g0 = 0; g0 < num_grid_w; g0++) { + GridAndStride gs; + gs.grid0 = g0; + gs.grid1 = g1; + gs.stride = stride; + grid_strides.push_back(gs); + } + } + } +} + +static void generate_proposals(std::vector grid_strides, const ncnn::Mat &pred, + float prob_threshold, std::vector &objects, int num_class) { + const int num_points = grid_strides.size(); + const int reg_max_1 = 16; + + for (int i = 0; i < num_points; i++) { + const float *scores = pred.row(i) + 4 * reg_max_1; + + // find label with max score + int label = -1; + float score = -FLT_MAX; + for (int k = 0; k < num_class; k++) { + float confidence = scores[k]; + if (confidence > score) { + label = k; + score = confidence; + } + } + float box_prob = sigmoid(score); + if (box_prob >= prob_threshold) { + ncnn::Mat bbox_pred(reg_max_1, 4, (void *) pred.row(i)); + { + ncnn::Layer *softmax = ncnn::create_layer("Softmax"); + + ncnn::ParamDict pd; + pd.set(0, 1); // axis + pd.set(1, 1); + softmax->load_param(pd); + + ncnn::Option opt; + opt.num_threads = 1; + opt.use_packing_layout = false; + + softmax->create_pipeline(opt); + + softmax->forward_inplace(bbox_pred, opt); + + softmax->destroy_pipeline(opt); + + delete softmax; + } + + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) { + float dis = 0.f; + const float *dis_after_sm = bbox_pred.row(k); + for (int l = 0; l < reg_max_1; l++) { + dis += l * dis_after_sm[l]; + } + + pred_ltrb[k] = dis * grid_strides[i].stride; + } + + float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; + float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = label; + obj.prob = box_prob; + + objects.push_back(obj); + } + } +} + +/***模型分割*************/ +static void matmul(const std::vector &bottom_blobs, ncnn::Mat &top_blob) { + ncnn::Option opt; + opt.num_threads = 2; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("MatMul"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 0);// axis + + op->load_param(pd); + + op->create_pipeline(opt); + std::vector top_blobs(1); + op->forward(bottom_blobs, top_blobs, opt); + top_blob = top_blobs[0]; + + op->destroy_pipeline(opt); + + delete op; +} + +static void sigmoid(ncnn::Mat &bottom) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Sigmoid"); + + op->create_pipeline(opt); + + // forward + + op->forward_inplace(bottom, opt); + op->destroy_pipeline(opt); + + delete op; +} + +static void reshape(const ncnn::Mat &in, ncnn::Mat &out, int c, int h, int w, int d) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Reshape"); + + // set param + ncnn::ParamDict pd; + + pd.set(0, w);// start + pd.set(1, h);// end + if (d > 0) + pd.set(11, d);//axes + pd.set(2, c);//axes + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void slice(const ncnn::Mat &in, ncnn::Mat &out, int start, int end, int axis) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Crop"); + + // set param + ncnn::ParamDict pd; + + ncnn::Mat axes = ncnn::Mat(1); + axes.fill(axis); + ncnn::Mat ends = ncnn::Mat(1); + ends.fill(end); + ncnn::Mat starts = ncnn::Mat(1); + starts.fill(start); + pd.set(9, starts);// start + pd.set(10, ends);// end + pd.set(11, axes);//axes + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void interp(const ncnn::Mat &in, const float &scale, const int &out_w, const int &out_h, + ncnn::Mat &out) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Interp"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 2);// resize_type + pd.set(1, scale);// height_scale + pd.set(2, scale);// width_scale + pd.set(3, out_h);// height + pd.set(4, out_w);// width + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void decode_mask(const ncnn::Mat &mask_feat, const int &img_w, const int &img_h, + const ncnn::Mat &mask_proto, const ncnn::Mat &in_pad, const int &wpad, + const int &hpad, ncnn::Mat &mask_pred_result) { + ncnn::Mat masks; + matmul(std::vector{mask_feat, mask_proto}, masks); + sigmoid(masks); + reshape(masks, masks, masks.h, in_pad.h / 4, in_pad.w / 4, 0); + slice(masks, mask_pred_result, (wpad / 2) / 4, (in_pad.w - wpad / 2) / 4, 2); + slice(mask_pred_result, mask_pred_result, (hpad / 2) / 4, (in_pad.h - hpad / 2) / 4, 1); + interp(mask_pred_result, 4.0, img_w, img_h, mask_pred_result); +} + +/***模型分割*************/ + +Yolo::Yolo() { + blob_pool_allocator.set_size_compare_ratio(0.f); + workspace_pool_allocator.set_size_compare_ratio(0.f); +} + +int +Yolo::load(const char *model_type, int target_size, const float *mean_values, + const float *norm_values, bool use_gpu) { + +} + +int +Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, + const float *_norm_values, bool use_gpu) { + yolo.clear(); + blob_pool_allocator.clear(); + workspace_pool_allocator.clear(); + + ncnn::set_cpu_powersave(2); + ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); + + yolo.opt = ncnn::Option(); + +#if NCNN_VULKAN + yolo.opt.use_vulkan_compute = use_gpu; +#endif + + yolo.opt.num_threads = ncnn::get_big_cpu_count(); + yolo.opt.blob_allocator = &blob_pool_allocator; + yolo.opt.workspace_allocator = &workspace_pool_allocator; + + char param_path[256]; + char model_path[256]; + //拼接模型名（路径） + sprintf(param_path, "%s.param", model_type); + sprintf(model_path, "%s.bin", model_type); + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); + + yolo.load_param(mgr, param_path); + yolo.load_model(mgr, model_path); + + target_size = _target_size; + mean_values[0] = _mean_values[0]; + mean_values[1] = _mean_values[1]; + mean_values[2] = _mean_values[2]; + norm_values[0] = _norm_values[0]; + norm_values[1] = _norm_values[1]; + norm_values[2] = _norm_values[2]; + + return 0; +} + +void Yolo::initNativeCallback(JavaVM *vm, jobject input, jlong nativeObjAddr, jobject pJobject) { + javaVM = vm; + + /** + * JNIEnv不支持跨线程调用 + * */ + JNIEnv *env; + vm->AttachCurrentThread(&env, nullptr); + //此时input转为output + j_output = env->NewGlobalRef(input); + + j_mat_addr = nativeObjAddr; + + j_callback = env->NewGlobalRef(pJobject); +} + +int Yolo::classify(const cv::Mat &rgb) { + if (state == 0) { + static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; + + int width = rgb.cols; + int height = rgb.rows; + + //把opencv Mat转为 ncnn Mat + ncnn::Mat in = ncnn::Mat::from_pixels(rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height); + + std::vector cls_scores; + { + in.substract_mean_normalize(mean_values, scale_values); + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in); + + ncnn::Mat out; + ex.extract("output", out); + + int output_size = out.w; + float float_buffer[output_size]; + for (int j = 0; j < out.w; j++) { + float_buffer[j] = out[j]; + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID(callback_clazz, "onClassify", "([F)V"); + + jfloatArray j_output_Data = env->NewFloatArray(output_size); + env->SetFloatArrayRegion(j_output_Data, 0, output_size, float_buffer); + + env->CallVoidMethod(j_callback, j_method_id, j_output_Data); + } + } + return 0; +} + +int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 1) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_BGR2RGB, width, + height, w, h); + + // pad to target_size rectangle + int wpad = (w + 31) / 32 * 32 - w; + int hpad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, + ncnn::BORDER_CONSTANT, 0.f); + + const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; + in_pad.substract_mean_normalize(0, norm_vals); + + + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (wpad / 2)) / scale; + float y0 = (objects[i].rect.y - (hpad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); + + for (auto item: objects) { + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + + /** + * Mat数据。 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 2) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale = 1.f; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize( + rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height, w, h + ); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border( + in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, + ncnn::BORDER_CONSTANT, 0.f + ); + + in_pad.substract_mean_normalize(0, norm_values); + + ncnn::Extractor ex = yolo.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, prob_threshold, proposals, 43); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + /** + * I: 整数类型（int） + * J: 长整数类型（long） + * D: 双精度浮点数类型（double） + * F: 单精度浮点数类型（float） + * Z: 布尔类型（boolean） + * C: 字符类型（char） + * B: 字节类型（byte） + * S: 短整数类型（short） + *
-----------------------------------------------
+ * Ljava/lang/Object;: 表示 Object 类型的引用 + * Ljava/lang/String;: 表示 String 类型的引用 + * L包名/类名;: 表示特定包名和类名的引用 + *
-----------------------------------------------
+ * 例如： + * int add(int a, int b): (II)I + * + * String concat(String str1, String str2): (Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String; + *
-----------------------------------------------
+ * [Ljava/lang/String;: 表示 String 类型的一维数组 + * */ + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Lcom/casic/br/ar/app/external/YoloResult;)V" + ); + + for (int i = 0; i < count; i++) { + auto item = objects[i]; + + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //回调 + env->CallVoidMethod(j_callback, j_method_id, j_output); + } + + /** + * Mat数据。 + *
-----------------------------------------------
+ * 通过内存地址赋值。Java层传入Mat对象内存地址，再通过C++给此地址赋值，Java即可得到内存地址的Mat矩阵数据 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::draw(cv::Mat &rgb, const std::vector &objects) { + static const char *class_names[] = { + "tripod", "tee", "person", + "shut-off valve", "hazard signs", "pressure tester", + "pressure gauge", "reflective clothing", "respirator masks", + "throat foil", "round-headed water gun", "safety signs", + "helmet", "security identification", "safety ropes", + "intercom", "pointed water gun", "switch", + "alarm device", "joint", "construction street signs", + "gas detectors", "hoses", "hose_rectangle", + "flow-meter", "fire hydrant box", "fire extinguisher", + "lighting equipment", "flame-out protection", "exposed wires", + "circuit diagram", "cordon", "regulator", + "length adjuster", "stickers", "across wires", + "road cones", "hose", "filter", + "distribution box", "long-shank valves", "valve", "ducts" + }; + + static const unsigned char colors[19][3] = { + {54, 67, 244}, + {99, 30, 233}, + {176, 39, 156}, + {183, 58, 103}, + {181, 81, 63}, + {243, 150, 33}, + {244, 169, 3}, + {212, 188, 0}, + {136, 150, 0}, + {80, 175, 76}, + {74, 195, 139}, + {57, 220, 205}, + {59, 235, 255}, + {7, 193, 255}, + {0, 152, 255}, + {34, 87, 255}, + {72, 85, 121}, + {158, 158, 158}, + {139, 125, 96} + }; + + int color_index = 0; + + for (const auto &obj: objects) { + const unsigned char *color = colors[color_index % 19]; + + color_index++; + + cv::Scalar cc(color[0], color[1], color[2]); + + cv::rectangle(rgb, obj.rect, cc, 2); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, + &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > rgb.cols) + x = rgb.cols - label_size.width; + + cv::Size size = cv::Size(label_size.width, label_size.height + baseLine); + cv::Rect rc = cv::Rect(cv::Point(x, y), size); + cv::rectangle(rgb, rc, cc, -1); + + cv::Scalar text_scalar = (color[0] + color[1] + color[2] >= 381) + ? cv::Scalar(0, 0, 0) + : cv::Scalar(255, 255, 255); + + + cv::putText(rgb, text, + cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, + 0.5, + text_scalar, 1 + ); + } + return 0; +} \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h new file mode 100644 index 0000000..1874609 --- /dev/null +++ b/app/src/main/cpp/yolo.h @@ -0,0 +1,107 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef YOLO_H +#define YOLO_H + +#include + +#include + +struct Object { + cv::Rect_ rect; + int label; + float prob; + cv::Mat mask; + std::vector mask_feat; +}; +struct GridAndStride { + int grid0; + int grid1; + int stride; +}; + +class Yolo { +public: + Yolo(); + + /** + * Yolo当前状态 + *
---------------
+ * 0 - 分类
+ * 1 - 分割
+ * 2 - 检测
+ * 3 - 绘制
+ * */ + int state = 0; + + int load(const char *model_type, + int target_size, + const float *mean_values, + const float *norm_values, + bool use_gpu = false); + + int load(AAssetManager *mgr, + const char *model_type, + int target_size, + const float *mean_values, + const float *norm_values, + bool use_gpu = false); + + void initNativeCallback(JavaVM *vm, jobject result, jlong nativeObjAddr, jobject pJobject); + + /** + * 分类 + * */ + int classify(const cv::Mat &rgb); + + /** + * 分割 + * */ + int partition(const cv::Mat &rgb, + std::vector &objects, + float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + + /** + * 检测 + * */ + int detect(const cv::Mat &rgb, + std::vector &objects, + float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + + int draw(cv::Mat &rgb, const std::vector &objects); + +private: + ncnn::Net yolo; + int target_size; + float mean_values[3]; + float norm_values[3]; + ncnn::UnlockedPoolAllocator blob_pool_allocator; + ncnn::PoolAllocator workspace_pool_allocator; + + /** + * 全局引用 + * */ + JavaVM *javaVM; + //输出结果类 + jobject j_output; + //Java传过来的Mat对象内存地址 + jlong j_mat_addr; + //回调类 + jobject j_callback; +}; + +#endif diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..9ddca05 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov8ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-2.4.13.7-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20240102-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/app/src/main/cpp/ndkcamera.cpp b/app/src/main/cpp/ndkcamera.cpp new file mode 100644 index 0000000..ed1e932 --- /dev/null +++ b/app/src/main/cpp/ndkcamera.cpp @@ -0,0 +1,699 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void *context, ACameraDevice *device) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void *context, ACameraDevice *device, int error) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void *context, AImageReader *reader) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage *image = nullptr; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = nullptr; + uint8_t *u_data = nullptr; + uint8_t *v_data = nullptr; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && + u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && + v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + auto *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *y_ptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + y_ptr[0] = y_data_ptr[0]; + y_ptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uv_ptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uv_ptr[0] = v_data_ptr[0]; + uv_ptr[1] = u_data_ptr[0]; + uv_ptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } + } + + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void *context, ACameraCaptureSession *session) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + ACameraCaptureFailure *failure) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, + failure); +} + +void onCaptureSequenceCompleted(void *context, ACameraCaptureSession *session, int sequenceId, + int64_t frameNumber) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", + session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void *context, ACameraCaptureSession *session, int sequenceId) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, + sequenceId); +} + +void onCaptureCompleted(void *context, ACameraCaptureSession *session, ACaptureRequest *request, + const ACameraMetadata *result) { +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() { + camera_facing = 0; + camera_orientation = 0; + + camera_manager = nullptr; + camera_device = nullptr; + image_reader = nullptr; + image_reader_surface = nullptr; + image_reader_target = nullptr; + capture_request = nullptr; + capture_session_output_container = nullptr; + capture_session_output = nullptr; + capture_session = nullptr; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() { + close(); + + if (image_reader) { + AImageReader_delete(image_reader); + image_reader = nullptr; + } + + if (image_reader_surface) { + ANativeWindow_release(image_reader_surface); + image_reader_surface = nullptr; + } +} + +int NdkCamera::open(int _camera_facing) { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList *camera_id_list = nullptr; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) { + const char *id = camera_id_list->cameraIds[i]; + ACameraMetadata *camera_metadata = nullptr; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t) e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = {0}; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int) e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), + camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, + &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, + capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, + &camera_capture_session_state_callbacks, + &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = nullptr; + camera_capture_session_capture_callbacks.onCaptureProgressed = nullptr; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = nullptr; + + ACameraCaptureSession_setRepeatingRequest(capture_session, + &camera_capture_session_capture_callbacks, 1, + &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() { + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = nullptr; + } + + if (camera_device) { + ACameraDevice_close(camera_device); + camera_device = nullptr; + } + + if (capture_session_output_container) { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = nullptr; + } + + if (capture_session_output) { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = nullptr; + } + + if (capture_request) { + ACaptureRequest_free(capture_request); + capture_request = nullptr; + } + + if (image_reader_target) { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = nullptr; + } + + if (camera_manager) { + ACameraManager_delete(camera_manager); + camera_manager = nullptr; + } +} + +void NdkCamera::on_image(const cv::Mat &rgb) const { +} + +void NdkCamera::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, + rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() { + sensor_manager = nullptr; + sensor_event_queue = nullptr; + accelerometer_sensor = nullptr; + win = nullptr; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, + ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() { + if (accelerometer_sensor) { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = nullptr; + } + + if (sensor_event_queue) { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = nullptr; + } + + if (win) { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow *_win) { + if (win) { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat &rgb) const { +} + +void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const { + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare( + ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, nullptr, nullptr, nullptr); + if (id == NDKCAMERAWINDOW_ID) { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) { + if (win_w * nv21_height > win_h * nv21_width) { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) { + if (win_w * nv21_width > win_h * nv21_height) { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } else { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) { + if (camera_orientation == 0 && accelerometer_orientation == 0) { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) { + rotate_type = 2; + } + } else { + if (final_orientation == 0) { + rotate_type = 1; + } + if (final_orientation == 90) { + rotate_type = 6; + } + if (final_orientation == 180) { + rotate_type = 3; + } + if (final_orientation == 270) { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char *srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char *dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, + rotate_type); + + const unsigned char *srcUV = + nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char *dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, + roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, + render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, + AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || + buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) { + for (int y = 0; y < render_h; y++) { + const auto *ptr = rgb_render.ptr(y); + unsigned char *outptr = (unsigned char *) buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/app/src/main/cpp/ndkcamera.h b/app/src/main/cpp/ndkcamera.h new file mode 100644 index 0000000..92abbaf --- /dev/null +++ b/app/src/main/cpp/ndkcamera.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef NDKCAMERA_H +#define NDKCAMERA_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +class NdkCamera { +public: + NdkCamera(); + + virtual ~NdkCamera(); + + // facing 0=front 1=back + int open(int camera_facing = 0); + + void close(); + + virtual void on_image(const cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + int camera_facing; + int camera_orientation; + +private: + ACameraManager *camera_manager; + ACameraDevice *camera_device; + AImageReader *image_reader; + ANativeWindow *image_reader_surface; + ACameraOutputTarget *image_reader_target; + ACaptureRequest *capture_request; + ACaptureSessionOutputContainer *capture_session_output_container; + ACaptureSessionOutput *capture_session_output; + ACameraCaptureSession *capture_session; +}; + +class NdkCameraWindow : public NdkCamera { +public: + NdkCameraWindow(); + + virtual ~NdkCameraWindow(); + + void set_window(ANativeWindow *win); + + virtual void on_image_render(cv::Mat &rgb) const; + + virtual void on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const; + +public: + mutable int accelerometer_orientation; + +private: + ASensorManager *sensor_manager; + mutable ASensorEventQueue *sensor_event_queue; + const ASensor *accelerometer_sensor; + ANativeWindow *win; +}; + +#endif // NDKCAMERA_H diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp new file mode 100644 index 0000000..4195087 --- /dev/null +++ b/app/src/main/cpp/yolo.cpp @@ -0,0 +1,853 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "yolo.h" + +#include +#include + +#include + +#include "cpu.h" + +static float fast_exp(float x) { + union { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; +} + +static float sigmoid(float x) { + return 1.0f / (1.0f + fast_exp(-x)); +} + +static float intersection_area(const Object &a, const Object &b) { + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); +} + +static void qsort_descent_inplace(std::vector &face_objects, int left, int right) { + int i = left; + int j = right; + float p = face_objects[(left + right) / 2].prob; + + while (i <= j) { + while (face_objects[i].prob > p) + i++; + + while (face_objects[j].prob < p) + j--; + + if (i <= j) { + // swap + std::swap(face_objects[i], face_objects[j]); + + i++; + j--; + } + } + + // #pragma omp parallel sections + { + // #pragma omp section + { + if (left < j) qsort_descent_inplace(face_objects, left, j); + } + // #pragma omp section + { + if (i < right) qsort_descent_inplace(face_objects, i, right); + } + } +} + +static void qsort_descent_inplace(std::vector &face_objects) { + if (face_objects.empty()) + return; + + qsort_descent_inplace(face_objects, 0, face_objects.size() - 1); +} + +static void nms_sorted_bboxes(const std::vector &face_objects, std::vector &picked, + float nms_threshold) { + picked.clear(); + + const int n = face_objects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) { + areas[i] = face_objects[i].rect.width * face_objects[i].rect.height; + } + + for (int i = 0; i < n; i++) { + const Object &a = face_objects[i]; + + int keep = 1; + for (int j = 0; j < (int) picked.size(); j++) { + const Object &b = face_objects[picked[j]]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } +} + +static void +generate_grids_and_stride(const int target_w, const int target_h, std::vector &strides, + std::vector &grid_strides) { + for (int stride: strides) { + int num_grid_w = target_w / stride; + int num_grid_h = target_h / stride; + for (int g1 = 0; g1 < num_grid_h; g1++) { + for (int g0 = 0; g0 < num_grid_w; g0++) { + GridAndStride gs; + gs.grid0 = g0; + gs.grid1 = g1; + gs.stride = stride; + grid_strides.push_back(gs); + } + } + } +} + +static void generate_proposals(std::vector grid_strides, const ncnn::Mat &pred, + float prob_threshold, std::vector &objects, int num_class) { + const int num_points = grid_strides.size(); + const int reg_max_1 = 16; + + for (int i = 0; i < num_points; i++) { + const float *scores = pred.row(i) + 4 * reg_max_1; + + // find label with max score + int label = -1; + float score = -FLT_MAX; + for (int k = 0; k < num_class; k++) { + float confidence = scores[k]; + if (confidence > score) { + label = k; + score = confidence; + } + } + float box_prob = sigmoid(score); + if (box_prob >= prob_threshold) { + ncnn::Mat bbox_pred(reg_max_1, 4, (void *) pred.row(i)); + { + ncnn::Layer *softmax = ncnn::create_layer("Softmax"); + + ncnn::ParamDict pd; + pd.set(0, 1); // axis + pd.set(1, 1); + softmax->load_param(pd); + + ncnn::Option opt; + opt.num_threads = 1; + opt.use_packing_layout = false; + + softmax->create_pipeline(opt); + + softmax->forward_inplace(bbox_pred, opt); + + softmax->destroy_pipeline(opt); + + delete softmax; + } + + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) { + float dis = 0.f; + const float *dis_after_sm = bbox_pred.row(k); + for (int l = 0; l < reg_max_1; l++) { + dis += l * dis_after_sm[l]; + } + + pred_ltrb[k] = dis * grid_strides[i].stride; + } + + float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; + float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = label; + obj.prob = box_prob; + + objects.push_back(obj); + } + } +} + +/***模型分割*************/ +static void matmul(const std::vector &bottom_blobs, ncnn::Mat &top_blob) { + ncnn::Option opt; + opt.num_threads = 2; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("MatMul"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 0);// axis + + op->load_param(pd); + + op->create_pipeline(opt); + std::vector top_blobs(1); + op->forward(bottom_blobs, top_blobs, opt); + top_blob = top_blobs[0]; + + op->destroy_pipeline(opt); + + delete op; +} + +static void sigmoid(ncnn::Mat &bottom) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Sigmoid"); + + op->create_pipeline(opt); + + // forward + + op->forward_inplace(bottom, opt); + op->destroy_pipeline(opt); + + delete op; +} + +static void reshape(const ncnn::Mat &in, ncnn::Mat &out, int c, int h, int w, int d) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Reshape"); + + // set param + ncnn::ParamDict pd; + + pd.set(0, w);// start + pd.set(1, h);// end + if (d > 0) + pd.set(11, d);//axes + pd.set(2, c);//axes + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void slice(const ncnn::Mat &in, ncnn::Mat &out, int start, int end, int axis) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Crop"); + + // set param + ncnn::ParamDict pd; + + ncnn::Mat axes = ncnn::Mat(1); + axes.fill(axis); + ncnn::Mat ends = ncnn::Mat(1); + ends.fill(end); + ncnn::Mat starts = ncnn::Mat(1); + starts.fill(start); + pd.set(9, starts);// start + pd.set(10, ends);// end + pd.set(11, axes);//axes + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void interp(const ncnn::Mat &in, const float &scale, const int &out_w, const int &out_h, + ncnn::Mat &out) { + ncnn::Option opt; + opt.num_threads = 4; + opt.use_fp16_storage = false; + opt.use_packing_layout = false; + + ncnn::Layer *op = ncnn::create_layer("Interp"); + + // set param + ncnn::ParamDict pd; + pd.set(0, 2);// resize_type + pd.set(1, scale);// height_scale + pd.set(2, scale);// width_scale + pd.set(3, out_h);// height + pd.set(4, out_w);// width + + op->load_param(pd); + + op->create_pipeline(opt); + + // forward + op->forward(in, out, opt); + + op->destroy_pipeline(opt); + + delete op; +} + +static void decode_mask(const ncnn::Mat &mask_feat, const int &img_w, const int &img_h, + const ncnn::Mat &mask_proto, const ncnn::Mat &in_pad, const int &wpad, + const int &hpad, ncnn::Mat &mask_pred_result) { + ncnn::Mat masks; + matmul(std::vector{mask_feat, mask_proto}, masks); + sigmoid(masks); + reshape(masks, masks, masks.h, in_pad.h / 4, in_pad.w / 4, 0); + slice(masks, mask_pred_result, (wpad / 2) / 4, (in_pad.w - wpad / 2) / 4, 2); + slice(mask_pred_result, mask_pred_result, (hpad / 2) / 4, (in_pad.h - hpad / 2) / 4, 1); + interp(mask_pred_result, 4.0, img_w, img_h, mask_pred_result); +} + +/***模型分割*************/ + +Yolo::Yolo() { + blob_pool_allocator.set_size_compare_ratio(0.f); + workspace_pool_allocator.set_size_compare_ratio(0.f); +} + +int +Yolo::load(const char *model_type, int target_size, const float *mean_values, + const float *norm_values, bool use_gpu) { + +} + +int +Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, + const float *_norm_values, bool use_gpu) { + yolo.clear(); + blob_pool_allocator.clear(); + workspace_pool_allocator.clear(); + + ncnn::set_cpu_powersave(2); + ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); + + yolo.opt = ncnn::Option(); + +#if NCNN_VULKAN + yolo.opt.use_vulkan_compute = use_gpu; +#endif + + yolo.opt.num_threads = ncnn::get_big_cpu_count(); + yolo.opt.blob_allocator = &blob_pool_allocator; + yolo.opt.workspace_allocator = &workspace_pool_allocator; + + char param_path[256]; + char model_path[256]; + //拼接模型名（路径） + sprintf(param_path, "%s.param", model_type); + sprintf(model_path, "%s.bin", model_type); + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); + + yolo.load_param(mgr, param_path); + yolo.load_model(mgr, model_path); + + target_size = _target_size; + mean_values[0] = _mean_values[0]; + mean_values[1] = _mean_values[1]; + mean_values[2] = _mean_values[2]; + norm_values[0] = _norm_values[0]; + norm_values[1] = _norm_values[1]; + norm_values[2] = _norm_values[2]; + + return 0; +} + +void Yolo::initNativeCallback(JavaVM *vm, jobject input, jlong nativeObjAddr, jobject pJobject) { + javaVM = vm; + + /** + * JNIEnv不支持跨线程调用 + * */ + JNIEnv *env; + vm->AttachCurrentThread(&env, nullptr); + //此时input转为output + j_output = env->NewGlobalRef(input); + + j_mat_addr = nativeObjAddr; + + j_callback = env->NewGlobalRef(pJobject); +} + +int Yolo::classify(const cv::Mat &rgb) { + if (state == 0) { + static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; + + int width = rgb.cols; + int height = rgb.rows; + + //把opencv Mat转为 ncnn Mat + ncnn::Mat in = ncnn::Mat::from_pixels(rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height); + + std::vector cls_scores; + { + in.substract_mean_normalize(mean_values, scale_values); + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in); + + ncnn::Mat out; + ex.extract("output", out); + + int output_size = out.w; + float float_buffer[output_size]; + for (int j = 0; j < out.w; j++) { + float_buffer[j] = out[j]; + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID(callback_clazz, "onClassify", "([F)V"); + + jfloatArray j_output_Data = env->NewFloatArray(output_size); + env->SetFloatArrayRegion(j_output_Data, 0, output_size, float_buffer); + + env->CallVoidMethod(j_callback, j_method_id, j_output_Data); + } + } + return 0; +} + +int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 1) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_BGR2RGB, width, + height, w, h); + + // pad to target_size rectangle + int wpad = (w + 31) / 32 * 32 - w; + int hpad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, + ncnn::BORDER_CONSTANT, 0.f); + + const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; + in_pad.substract_mean_normalize(0, norm_vals); + + + ncnn::Extractor ex = yolo.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (wpad / 2)) / scale; + float y0 = (objects[i].rect.y - (hpad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); + + for (auto item: objects) { + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + + /** + * Mat数据。 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (state == 2) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale = 1.f; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize( + rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height, w, h + ); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border( + in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, + ncnn::BORDER_CONSTANT, 0.f + ); + + in_pad.substract_mean_normalize(0, norm_values); + + ncnn::Extractor ex = yolo.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, prob_threshold, proposals, 43); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + /** + * 回调给Java/Kotlin层 + * */ + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jclass output_clazz = env->GetObjectClass(j_output); + /** + * I: 整数类型（int） + * J: 长整数类型（long） + * D: 双精度浮点数类型（double） + * F: 单精度浮点数类型（float） + * Z: 布尔类型（boolean） + * C: 字符类型（char） + * B: 字节类型（byte） + * S: 短整数类型（short） + *
-----------------------------------------------
+ * Ljava/lang/Object;: 表示 Object 类型的引用 + * Ljava/lang/String;: 表示 String 类型的引用 + * L包名/类名;: 表示特定包名和类名的引用 + *
-----------------------------------------------
+ * 例如： + * int add(int a, int b): (II)I + * + * String concat(String str1, String str2): (Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String; + *
-----------------------------------------------
+ * [Ljava/lang/String;: 表示 String 类型的一维数组 + * */ + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Lcom/casic/br/ar/app/external/YoloResult;)V" + ); + + for (int i = 0; i < count; i++) { + auto item = objects[i]; + + jfieldID type = env->GetFieldID(output_clazz, "type", "I"); + env->SetIntField(j_output, type, item.label); + + jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + jfloatArray rectArray = env->NewFloatArray(4); + env->SetFloatArrayRegion(rectArray, 0, 4, array); + env->SetObjectField(j_output, position, rectArray); + + jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); + env->SetFloatField(j_output, prob, item.prob); + + //回调 + env->CallVoidMethod(j_callback, j_method_id, j_output); + } + + /** + * Mat数据。 + *
-----------------------------------------------
+ * 通过内存地址赋值。Java层传入Mat对象内存地址，再通过C++给此地址赋值，Java即可得到内存地址的Mat矩阵数据 + * */ + auto *res = (cv::Mat *) j_mat_addr; + res->create(rgb.rows, rgb.cols, rgb.type()); + memcpy(res->data, rgb.data, rgb.rows * rgb.step); + } + return 0; +} + +int Yolo::draw(cv::Mat &rgb, const std::vector &objects) { + static const char *class_names[] = { + "tripod", "tee", "person", + "shut-off valve", "hazard signs", "pressure tester", + "pressure gauge", "reflective clothing", "respirator masks", + "throat foil", "round-headed water gun", "safety signs", + "helmet", "security identification", "safety ropes", + "intercom", "pointed water gun", "switch", + "alarm device", "joint", "construction street signs", + "gas detectors", "hoses", "hose_rectangle", + "flow-meter", "fire hydrant box", "fire extinguisher", + "lighting equipment", "flame-out protection", "exposed wires", + "circuit diagram", "cordon", "regulator", + "length adjuster", "stickers", "across wires", + "road cones", "hose", "filter", + "distribution box", "long-shank valves", "valve", "ducts" + }; + + static const unsigned char colors[19][3] = { + {54, 67, 244}, + {99, 30, 233}, + {176, 39, 156}, + {183, 58, 103}, + {181, 81, 63}, + {243, 150, 33}, + {244, 169, 3}, + {212, 188, 0}, + {136, 150, 0}, + {80, 175, 76}, + {74, 195, 139}, + {57, 220, 205}, + {59, 235, 255}, + {7, 193, 255}, + {0, 152, 255}, + {34, 87, 255}, + {72, 85, 121}, + {158, 158, 158}, + {139, 125, 96} + }; + + int color_index = 0; + + for (const auto &obj: objects) { + const unsigned char *color = colors[color_index % 19]; + + color_index++; + + cv::Scalar cc(color[0], color[1], color[2]); + + cv::rectangle(rgb, obj.rect, cc, 2); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, + &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > rgb.cols) + x = rgb.cols - label_size.width; + + cv::Size size = cv::Size(label_size.width, label_size.height + baseLine); + cv::Rect rc = cv::Rect(cv::Point(x, y), size); + cv::rectangle(rgb, rc, cc, -1); + + cv::Scalar text_scalar = (color[0] + color[1] + color[2] >= 381) + ? cv::Scalar(0, 0, 0) + : cv::Scalar(255, 255, 255); + + + cv::putText(rgb, text, + cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, + 0.5, + text_scalar, 1 + ); + } + return 0; +} \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h new file mode 100644 index 0000000..1874609 --- /dev/null +++ b/app/src/main/cpp/yolo.h @@ -0,0 +1,107 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef YOLO_H +#define YOLO_H + +#include + +#include + +struct Object { + cv::Rect_ rect; + int label; + float prob; + cv::Mat mask; + std::vector mask_feat; +}; +struct GridAndStride { + int grid0; + int grid1; + int stride; +}; + +class Yolo { +public: + Yolo(); + + /** + * Yolo当前状态 + *
---------------
+ * 0 - 分类
+ * 1 - 分割
+ * 2 - 检测
+ * 3 - 绘制
+ * */ + int state = 0; + + int load(const char *model_type, + int target_size, + const float *mean_values, + const float *norm_values, + bool use_gpu = false); + + int load(AAssetManager *mgr, + const char *model_type, + int target_size, + const float *mean_values, + const float *norm_values, + bool use_gpu = false); + + void initNativeCallback(JavaVM *vm, jobject result, jlong nativeObjAddr, jobject pJobject); + + /** + * 分类 + * */ + int classify(const cv::Mat &rgb); + + /** + * 分割 + * */ + int partition(const cv::Mat &rgb, + std::vector &objects, + float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + + /** + * 检测 + * */ + int detect(const cv::Mat &rgb, + std::vector &objects, + float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + + int draw(cv::Mat &rgb, const std::vector &objects); + +private: + ncnn::Net yolo; + int target_size; + float mean_values[3]; + float norm_values[3]; + ncnn::UnlockedPoolAllocator blob_pool_allocator; + ncnn::PoolAllocator workspace_pool_allocator; + + /** + * 全局引用 + * */ + JavaVM *javaVM; + //输出结果类 + jobject j_output; + //Java传过来的Mat对象内存地址 + jlong j_mat_addr; + //回调类 + jobject j_callback; +}; + +#endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp new file mode 100644 index 0000000..4e0e2d3 --- /dev/null +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -0,0 +1,265 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include + +#include "yolo.h" + +#include "ndkcamera.h" + +#include +#include + +#if __ARM_NEON +#include +#endif // __ARM_NEON + +static int draw_unsupported(cv::Mat &rgb) { + const char text[] = "unsupported"; + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 1.0, 1, &baseLine); + + int y = (rgb.rows - label_size.height) / 2; + int x = (rgb.cols - label_size.width) / 2; + + cv::rectangle(rgb, cv::Rect(cv::Point(x, y), + cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(rgb, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 0, 0)); + + return 0; +} + +static int draw_fps(cv::Mat &rgb) { + // resolve moving average + double avg_fps = 0.f; + { + static double t0 = 0.f; + static double fps_history[10] = {0.f}; + + double t1 = ncnn::get_current_time(); + if (t0 == 0.f) { + t0 = t1; + return 0; + } + + double fps = 1000.f / (t1 - t0); + t0 = t1; + + for (int i = 9; i >= 1; i--) { + fps_history[i] = fps_history[i - 1]; + } + fps_history[0] = fps; + + if (fps_history[9] == 0.f) { + return 0; + } + + for (double i: fps_history) { + avg_fps += i; + } + avg_fps /= 10.f; + } + + char text[32]; + sprintf(text, "FPS=%.2f", avg_fps); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int y = 0; + int x = rgb.cols - label_size.width; + + cv::rectangle(rgb, + cv::Rect(cv::Point(x, y), + cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(rgb, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); + + return 0; +} + +static Yolo *g_yolo = nullptr; +static ncnn::Mutex lock; +static JavaVM *javaVM = nullptr; + +class MyNdkCamera : public NdkCameraWindow { +public: + void on_image_render(cv::Mat &rgb) const override; +}; + +void MyNdkCamera::on_image_render(cv::Mat &rgb) const { + // nanodet + { + ncnn::MutexLockGuard g(lock); + + if (g_yolo) { + //分类 + g_yolo->classify(rgb); + + std::vector objects; + + //分割 + g_yolo->partition(rgb, objects); + + //检测 + g_yolo->detect(rgb, objects); + } else { + draw_unsupported(rgb); + } + } +} + +static MyNdkCamera *g_camera = nullptr; + +extern "C" { +JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); + + javaVM = vm; + + g_camera = new MyNdkCamera; + + return JNI_VERSION_1_4; +} + +JNIEXPORT void JNI_OnUnload(JavaVM *vm, void *reserved) { + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnUnload"); + + { + ncnn::MutexLockGuard g(lock); + + delete g_yolo; + g_yolo = nullptr; + } + + delete g_camera; + g_camera = nullptr; +} + +JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, + jint model_id, jint processor) { + if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + return JNI_FALSE; + } + + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + //分割、分类、检测 + const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; + + const int target_sizes[] = {320, 320, 320}; + + const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} + }; + + const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} + }; + + const char *model_type = model_types[(int) model_id]; + int target_size = target_sizes[(int) model_id]; + bool use_gpu = (int) processor == 1; + + // reload + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[(int) model_id], + norm_values[(int) model_id], use_gpu + ); + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { + if (facing < 0 || facing > 1) + return JNI_FALSE; + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "openCamera %d", facing); + + g_camera->open((int) facing); + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_closeCamera(JNIEnv *env, jobject thiz) { + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "closeCamera"); + + g_camera->close(); + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_setOutputWindow(JNIEnv *env, jobject thiz, jobject surface, + jobject input, jlong nativeObjAddr, + jobject native_callback) { + ANativeWindow *win = ANativeWindow_fromSurface(env, surface); + + g_camera->set_window(win); + + g_yolo->initNativeCallback(javaVM, input, nativeObjAddr, native_callback); + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, jint yolo_state) { + g_yolo->state = yolo_state; + return JNI_TRUE; +} + +JNIEXPORT jint JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { + return g_yolo->state; +} +} \ No newline at end of file