diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt index a73a2b3..a1a1f98 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt @@ -21,5 +21,5 @@ /** * 自由检测 */ - fun onFreeDetect(output: ArrayList) + fun onFreeDetect(segmentationOutput: ArrayList, detectOutput: ArrayList) } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt index a73a2b3..a1a1f98 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt @@ -21,5 +21,5 @@ /** * 自由检测 */ - fun onFreeDetect(output: ArrayList) + fun onFreeDetect(segmentationOutput: ArrayList, detectOutput: ArrayList) } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt index 05814dd..4333a39 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt @@ -32,6 +32,13 @@ external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun freeModeLoadModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean + + /** * @param facing 相机 0-前置镜头,1-后置镜头 * */ external fun openCamera(facing: Int): Boolean diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt index a73a2b3..a1a1f98 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt @@ -21,5 +21,5 @@ /** * 自由检测 */ - fun onFreeDetect(output: ArrayList) + fun onFreeDetect(segmentationOutput: ArrayList, detectOutput: ArrayList) } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt index 05814dd..4333a39 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt @@ -32,6 +32,13 @@ external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun freeModeLoadModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean + + /** * @param facing 相机 0-前置镜头,1-后置镜头 * */ external fun openCamera(facing: Int): Boolean diff --git a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt index d365fb8..69090db 100644 --- a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt +++ b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt @@ -316,204 +316,200 @@ override fun onSegmentation( segmentationOutput: ArrayList, detectOutput: ArrayList ) { - if (detectOutput.isEmpty()) { - binding.detectView.updateTargetPosition() - } else { - val segmentationResults = ArrayList() - segmentationOutput.forEach { - segmentationResults.add(it.convert2YoloResult(this)) - } - - val detectResults = ArrayList() - detectOutput.forEach { - detectResults.add(it.convert2YoloResult(this)) - } - - binding.detectView.updateTargetPosition(segmentationResults, detectResults) - - lifecycleScope.launch(Dispatchers.Main) { - detectResults.forEach { - val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] - if (label.isInScene(RuntimeCache.sceneName)) { - targetSet.add(label) - } - } - } - - //只显示隐患 - var warnings = "" - when (RuntimeCache.sceneName) { - "配电箱" -> { - if (!detectResults.isContains(16)) { - warnings = "配电箱无警示标识" - alarmCode = "DistributionBoxHasNoWarningSign" - } - - if (!detectResults.isContains(36)) { - warnings = "配电箱内无电路图" - alarmCode = "DistributionBoxHasNoCircuitDiagram" - } - - if (detectResults.isContains(35)) { - warnings = "配电箱内部电线裸露" - alarmCode = "" - } - - if (detectResults.isContains(41)) { - warnings = "配电箱有跨电线" - alarmCode = "DistributionBoxHasNoJumperWire" - } - - if (segmentationResults.isNotEmpty()) { - warnings = "配电箱内电线杂乱" - alarmCode = "DistributionBoxHasTangledWire" - } - } - - "有限空间作业" -> { - if (detectResults.isContains(3)) { - if (!detectResults.isContains(15)) { - warnings = "未佩戴安全帽" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" - } - - if (!detectResults.isContains(20)) { - warnings = "未着工服" - alarmCode = "ConfinedSpaceHasNoWorkerClothes" - } - - if (!detectResults.isContains(15) && !detectResults.isContains(17)) { - warnings = "没有佩戴安全带、安全绳" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" - } - } - - if (!detectResults.isContains(9)) { - warnings = "未发现呼吸防护设备" - alarmCode = "ConfinedSpaceHasNoWorkerMask" - } - - if (!detectResults.isContains(42) && !detectResults.isContains(37)) { - warnings = "未发现路锥、警戒线" - alarmCode = "ConfinedSpaceHasNoEnclosure" - } - - if (!detectResults.isContains(14)) { - warnings = "未发现安全告知牌" - alarmCode = "ConfinedSpaceHasNoWarningSign" - } - - if (!detectResults.isContains(51)) { - warnings = "未发现通风设备" - alarmCode = "ConfinedSpaceHasNoAirSupply" - } - - if (!detectResults.isContains(33)) { - warnings = "未发现井下照明设备" - alarmCode = "ConfinedSpaceHasNoLighting" - } - - if (!detectResults.isContains(18)) { - warnings = "未发现井下对讲设备" - alarmCode = "ConfinedSpaceHasNoIntercom" - } - - if (!detectResults.isContains(0)) { - warnings = "未发现施工三脚架" - alarmCode = "ConfinedSpaceHasNoTripod" - } - - if (!detectResults.isContains(25)) { - warnings = "未发现气体检测仪" - alarmCode = "ConfinedSpaceHasNoGasDetector" - } - } - - "非居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - - "居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - } - - if (warnings == "") { - return - } - - //显示弹框 - val message = weakReferenceHandler.obtainMessage() - message.what = 2024082902 - message.obj = warnings - weakReferenceHandler.sendMessage(message) + val segmentationResults = ArrayList() + segmentationOutput.forEach { + segmentationResults.add(it.convert2YoloResult(this)) } + + val detectResults = ArrayList() + detectOutput.forEach { + detectResults.add(it.convert2YoloResult(this)) + } + + binding.detectView.updateTargetPosition(segmentationResults, detectResults) + + lifecycleScope.launch(Dispatchers.Main) { + detectResults.forEach { + val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] + if (label.isInScene(RuntimeCache.sceneName)) { + targetSet.add(label) + } + } + } + + //只显示隐患 + var warnings = "" + when (RuntimeCache.sceneName) { + "配电箱" -> { + if (!detectResults.isContains(16)) { + warnings = "配电箱无警示标识" + alarmCode = "DistributionBoxHasNoWarningSign" + } + + if (!detectResults.isContains(36)) { + warnings = "配电箱内无电路图" + alarmCode = "DistributionBoxHasNoCircuitDiagram" + } + + if (detectResults.isContains(35)) { + warnings = "配电箱内部电线裸露" + alarmCode = "" + } + + if (detectResults.isContains(41)) { + warnings = "配电箱有跨电线" + alarmCode = "DistributionBoxHasNoJumperWire" + } + + if (segmentationResults.isNotEmpty()) { + warnings = "配电箱内电线杂乱" + alarmCode = "DistributionBoxHasTangledWire" + } + } + + "有限空间作业" -> { + if (detectResults.isContains(3)) { + if (!detectResults.isContains(15)) { + warnings = "未佩戴安全帽" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" + } + + if (!detectResults.isContains(20)) { + warnings = "未着工服" + alarmCode = "ConfinedSpaceHasNoWorkerClothes" + } + + if (!detectResults.isContains(15) && !detectResults.isContains(17)) { + warnings = "没有佩戴安全带、安全绳" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" + } + } + + if (!detectResults.isContains(9)) { + warnings = "未发现呼吸防护设备" + alarmCode = "ConfinedSpaceHasNoWorkerMask" + } + + if (!detectResults.isContains(42) && !detectResults.isContains(37)) { + warnings = "未发现路锥、警戒线" + alarmCode = "ConfinedSpaceHasNoEnclosure" + } + + if (!detectResults.isContains(14)) { + warnings = "未发现安全告知牌" + alarmCode = "ConfinedSpaceHasNoWarningSign" + } + + if (!detectResults.isContains(51)) { + warnings = "未发现通风设备" + alarmCode = "ConfinedSpaceHasNoAirSupply" + } + + if (!detectResults.isContains(33)) { + warnings = "未发现井下照明设备" + alarmCode = "ConfinedSpaceHasNoLighting" + } + + if (!detectResults.isContains(18)) { + warnings = "未发现井下对讲设备" + alarmCode = "ConfinedSpaceHasNoIntercom" + } + + if (!detectResults.isContains(0)) { + warnings = "未发现施工三脚架" + alarmCode = "ConfinedSpaceHasNoTripod" + } + + if (!detectResults.isContains(25)) { + warnings = "未发现气体检测仪" + alarmCode = "ConfinedSpaceHasNoGasDetector" + } + } + + "非居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + + "居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + } + + if (warnings == "") { + return + } + + //显示弹框 + val message = weakReferenceHandler.obtainMessage() + message.what = 2024082902 + message.obj = warnings + weakReferenceHandler.sendMessage(message) } override fun onDetect(output: ArrayList) { @@ -539,7 +535,9 @@ weakReferenceHandler.sendMessage(message) } - override fun onFreeDetect(output: ArrayList) { + override fun onFreeDetect( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { } diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt index a73a2b3..a1a1f98 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt @@ -21,5 +21,5 @@ /** * 自由检测 */ - fun onFreeDetect(output: ArrayList) + fun onFreeDetect(segmentationOutput: ArrayList, detectOutput: ArrayList) } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt index 05814dd..4333a39 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt @@ -32,6 +32,13 @@ external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun freeModeLoadModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean + + /** * @param facing 相机 0-前置镜头,1-后置镜头 * */ external fun openCamera(facing: Int): Boolean diff --git a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt index d365fb8..69090db 100644 --- a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt +++ b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt @@ -316,204 +316,200 @@ override fun onSegmentation( segmentationOutput: ArrayList, detectOutput: ArrayList ) { - if (detectOutput.isEmpty()) { - binding.detectView.updateTargetPosition() - } else { - val segmentationResults = ArrayList() - segmentationOutput.forEach { - segmentationResults.add(it.convert2YoloResult(this)) - } - - val detectResults = ArrayList() - detectOutput.forEach { - detectResults.add(it.convert2YoloResult(this)) - } - - binding.detectView.updateTargetPosition(segmentationResults, detectResults) - - lifecycleScope.launch(Dispatchers.Main) { - detectResults.forEach { - val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] - if (label.isInScene(RuntimeCache.sceneName)) { - targetSet.add(label) - } - } - } - - //只显示隐患 - var warnings = "" - when (RuntimeCache.sceneName) { - "配电箱" -> { - if (!detectResults.isContains(16)) { - warnings = "配电箱无警示标识" - alarmCode = "DistributionBoxHasNoWarningSign" - } - - if (!detectResults.isContains(36)) { - warnings = "配电箱内无电路图" - alarmCode = "DistributionBoxHasNoCircuitDiagram" - } - - if (detectResults.isContains(35)) { - warnings = "配电箱内部电线裸露" - alarmCode = "" - } - - if (detectResults.isContains(41)) { - warnings = "配电箱有跨电线" - alarmCode = "DistributionBoxHasNoJumperWire" - } - - if (segmentationResults.isNotEmpty()) { - warnings = "配电箱内电线杂乱" - alarmCode = "DistributionBoxHasTangledWire" - } - } - - "有限空间作业" -> { - if (detectResults.isContains(3)) { - if (!detectResults.isContains(15)) { - warnings = "未佩戴安全帽" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" - } - - if (!detectResults.isContains(20)) { - warnings = "未着工服" - alarmCode = "ConfinedSpaceHasNoWorkerClothes" - } - - if (!detectResults.isContains(15) && !detectResults.isContains(17)) { - warnings = "没有佩戴安全带、安全绳" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" - } - } - - if (!detectResults.isContains(9)) { - warnings = "未发现呼吸防护设备" - alarmCode = "ConfinedSpaceHasNoWorkerMask" - } - - if (!detectResults.isContains(42) && !detectResults.isContains(37)) { - warnings = "未发现路锥、警戒线" - alarmCode = "ConfinedSpaceHasNoEnclosure" - } - - if (!detectResults.isContains(14)) { - warnings = "未发现安全告知牌" - alarmCode = "ConfinedSpaceHasNoWarningSign" - } - - if (!detectResults.isContains(51)) { - warnings = "未发现通风设备" - alarmCode = "ConfinedSpaceHasNoAirSupply" - } - - if (!detectResults.isContains(33)) { - warnings = "未发现井下照明设备" - alarmCode = "ConfinedSpaceHasNoLighting" - } - - if (!detectResults.isContains(18)) { - warnings = "未发现井下对讲设备" - alarmCode = "ConfinedSpaceHasNoIntercom" - } - - if (!detectResults.isContains(0)) { - warnings = "未发现施工三脚架" - alarmCode = "ConfinedSpaceHasNoTripod" - } - - if (!detectResults.isContains(25)) { - warnings = "未发现气体检测仪" - alarmCode = "ConfinedSpaceHasNoGasDetector" - } - } - - "非居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - - "居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - } - - if (warnings == "") { - return - } - - //显示弹框 - val message = weakReferenceHandler.obtainMessage() - message.what = 2024082902 - message.obj = warnings - weakReferenceHandler.sendMessage(message) + val segmentationResults = ArrayList() + segmentationOutput.forEach { + segmentationResults.add(it.convert2YoloResult(this)) } + + val detectResults = ArrayList() + detectOutput.forEach { + detectResults.add(it.convert2YoloResult(this)) + } + + binding.detectView.updateTargetPosition(segmentationResults, detectResults) + + lifecycleScope.launch(Dispatchers.Main) { + detectResults.forEach { + val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] + if (label.isInScene(RuntimeCache.sceneName)) { + targetSet.add(label) + } + } + } + + //只显示隐患 + var warnings = "" + when (RuntimeCache.sceneName) { + "配电箱" -> { + if (!detectResults.isContains(16)) { + warnings = "配电箱无警示标识" + alarmCode = "DistributionBoxHasNoWarningSign" + } + + if (!detectResults.isContains(36)) { + warnings = "配电箱内无电路图" + alarmCode = "DistributionBoxHasNoCircuitDiagram" + } + + if (detectResults.isContains(35)) { + warnings = "配电箱内部电线裸露" + alarmCode = "" + } + + if (detectResults.isContains(41)) { + warnings = "配电箱有跨电线" + alarmCode = "DistributionBoxHasNoJumperWire" + } + + if (segmentationResults.isNotEmpty()) { + warnings = "配电箱内电线杂乱" + alarmCode = "DistributionBoxHasTangledWire" + } + } + + "有限空间作业" -> { + if (detectResults.isContains(3)) { + if (!detectResults.isContains(15)) { + warnings = "未佩戴安全帽" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" + } + + if (!detectResults.isContains(20)) { + warnings = "未着工服" + alarmCode = "ConfinedSpaceHasNoWorkerClothes" + } + + if (!detectResults.isContains(15) && !detectResults.isContains(17)) { + warnings = "没有佩戴安全带、安全绳" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" + } + } + + if (!detectResults.isContains(9)) { + warnings = "未发现呼吸防护设备" + alarmCode = "ConfinedSpaceHasNoWorkerMask" + } + + if (!detectResults.isContains(42) && !detectResults.isContains(37)) { + warnings = "未发现路锥、警戒线" + alarmCode = "ConfinedSpaceHasNoEnclosure" + } + + if (!detectResults.isContains(14)) { + warnings = "未发现安全告知牌" + alarmCode = "ConfinedSpaceHasNoWarningSign" + } + + if (!detectResults.isContains(51)) { + warnings = "未发现通风设备" + alarmCode = "ConfinedSpaceHasNoAirSupply" + } + + if (!detectResults.isContains(33)) { + warnings = "未发现井下照明设备" + alarmCode = "ConfinedSpaceHasNoLighting" + } + + if (!detectResults.isContains(18)) { + warnings = "未发现井下对讲设备" + alarmCode = "ConfinedSpaceHasNoIntercom" + } + + if (!detectResults.isContains(0)) { + warnings = "未发现施工三脚架" + alarmCode = "ConfinedSpaceHasNoTripod" + } + + if (!detectResults.isContains(25)) { + warnings = "未发现气体检测仪" + alarmCode = "ConfinedSpaceHasNoGasDetector" + } + } + + "非居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + + "居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + } + + if (warnings == "") { + return + } + + //显示弹框 + val message = weakReferenceHandler.obtainMessage() + message.what = 2024082902 + message.obj = warnings + weakReferenceHandler.sendMessage(message) } override fun onDetect(output: ArrayList) { @@ -539,7 +535,9 @@ weakReferenceHandler.sendMessage(message) } - override fun onFreeDetect(output: ArrayList) { + override fun onFreeDetect( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { } diff --git a/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt b/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt index f4e2922..9b43826 100644 --- a/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt +++ b/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt @@ -21,6 +21,7 @@ class FreeModeActivity : KotlinBaseActivity(), SurfaceHolder.Callback, INativeCallback { + private val kTag = "FreeModeActivity" private val yolov8ncnn by lazy { Yolov8ncnn() } override fun initEvent() { @@ -33,10 +34,7 @@ OpenCVNativeLoader().init() //加载算法模型 - yolov8ncnn.loadModel( - assets, 1, useGpu = false, - useClassify = false, useSegmentation = false, useDetect = true - ) + yolov8ncnn.freeModeLoadModel(assets, intArrayOf(0, 1), false) binding.surfaceView.holder.setFormat(PixelFormat.RGBA_8888) binding.surfaceView.holder.addCallback(this) @@ -78,13 +76,20 @@ } - override fun onFreeDetect(output: ArrayList) { - //转成泛型集合 - val results = ArrayList() - output.forEach { - results.add(it.convert2YoloResult(this)) + override fun onFreeDetect( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + val segmentationResults = ArrayList() + segmentationOutput.forEach { + segmentationResults.add(it.convert2YoloResult(this)) } - binding.detectView.updateTargetPosition(results) + + val detectResults = ArrayList() + detectOutput.forEach { + detectResults.add(it.convert2YoloResult(this)) + } + + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun surfaceCreated(holder: SurfaceHolder) {} diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 6cc45af..c8f6434 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -444,13 +444,6 @@ JNIEnv *env; vm->AttachCurrentThread(&env, nullptr); j_mat_addr = nativeObjAddr; - //nativeObjAddr 有两类值,一种是内存地址(检查模式),一种是常量 1L (自由模式) - if (nativeObjAddr == 1L) { - j_mode = 0; - } else { - j_mode = 1; - } - j_callback = env->NewGlobalRef(pJobject); } @@ -816,10 +809,133 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - if (j_mode == 0) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;)V" - ); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" + ); + for (const auto &item: objects) { + char index[16]; + sprintf(index, "%d", item.label); + + //add + env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + } + //回调 + env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); + } + return 0; +} + +int Yolo::free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 4) { + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 32 + int w = width; + int h = height; + float scale; + if (w > h) { + scale = (float) target_size / w; + w = target_size; + h = h * scale; + } else { + scale = (float) target_size / h; + h = target_size; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, + height, w, h); + + // pad to target_size rectangle + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); + + in_pad.substract_mean_normalize(0, norm_values); + + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onFreeDetect", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); + + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); + + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); + + ncnn::Mat out; + ex.extract("output", out); + + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); + + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + + qsort_descent_inplace(proposals); + + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } for (const auto &item: objects) { auto rect = item.rect; @@ -835,25 +951,85 @@ env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, result_array); + env->CallBooleanMethod(segment_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); - } else if (j_mode == 1) { - jmethodID j_method_id = env->GetMethodID( - callback_clazz, "onDetect", "(Ljava/util/ArrayList;)V" - ); + } + + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); + + ex.input("images", in_pad); + + std::vector proposals; + + ncnn::Mat out; + ex.extract("output", out); + + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 50); + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); for (const auto &item: objects) { - char index[16]; - sprintf(index, "%d", item.label); + auto rect = item.rect; + + float array[5]; + array[0] = rect.x; + array[1] = rect.y; + array[2] = rect.x + rect.width; + array[3] = rect.y + rect.height; + array[4] = (float) item.label; + + jfloatArray result_array = env->NewFloatArray(5); + env->SetFloatArrayRegion(result_array, 0, 5, array); //add - env->CallBooleanMethod(arraylist_obj, arraylist_add, env->NewStringUTF(index)); + env->CallBooleanMethod(detect_array_obj, arraylist_add, result_array); } - //回调 - env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); } + + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } \ No newline at end of file diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index cecba41..da8acf2 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -43,7 +43,7 @@ * 1 - 分类
* 2 - 分割
* 3 - 检测
- * 4 - 绘制
+ * 4 - 自由检测
* */ int j_state = 0; @@ -59,8 +59,6 @@ void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); - void initFreeModeNativeCallback(JavaVM *vm, jobject pJobject); - /** * 分类 * */ @@ -78,6 +76,12 @@ int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); + /** + * 自由模式下的检测 + * */ + int free_detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); + private: //分类 ncnn::Net yolo_c; @@ -99,8 +103,6 @@ jlong j_mat_addr; //回调类 jobject j_callback; - //工作模式,0-自由,1-普通 - jint j_mode; }; #endif diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index e466653..d537de5 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -34,7 +34,9 @@ #include #if __ARM_NEON + #include + #endif // __ARM_NEON static int draw_unsupported(cv::Mat &rgb) { @@ -132,6 +134,9 @@ //检测 g_yolo->detect(rgb, objects); + + //自由检测 + g_yolo->free_detect(rgb, objects); } else { draw_unsupported(rgb); } @@ -291,6 +296,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_ar_app_external_Yolov8ncnn_freeModeLoadModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 4; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_ar_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; diff --git a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt index a73a2b3..a1a1f98 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/INativeCallback.kt @@ -21,5 +21,5 @@ /** * 自由检测 */ - fun onFreeDetect(output: ArrayList) + fun onFreeDetect(segmentationOutput: ArrayList, detectOutput: ArrayList) } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt index 05814dd..4333a39 100644 --- a/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/ar/app/external/Yolov8ncnn.kt @@ -32,6 +32,13 @@ external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun freeModeLoadModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean + + /** * @param facing 相机 0-前置镜头,1-后置镜头 * */ external fun openCamera(facing: Int): Boolean diff --git a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt index d365fb8..69090db 100644 --- a/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt +++ b/app/src/main/java/com/casic/br/ar/app/view/CheckModeActivity.kt @@ -316,204 +316,200 @@ override fun onSegmentation( segmentationOutput: ArrayList, detectOutput: ArrayList ) { - if (detectOutput.isEmpty()) { - binding.detectView.updateTargetPosition() - } else { - val segmentationResults = ArrayList() - segmentationOutput.forEach { - segmentationResults.add(it.convert2YoloResult(this)) - } - - val detectResults = ArrayList() - detectOutput.forEach { - detectResults.add(it.convert2YoloResult(this)) - } - - binding.detectView.updateTargetPosition(segmentationResults, detectResults) - - lifecycleScope.launch(Dispatchers.Main) { - detectResults.forEach { - val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] - if (label.isInScene(RuntimeCache.sceneName)) { - targetSet.add(label) - } - } - } - - //只显示隐患 - var warnings = "" - when (RuntimeCache.sceneName) { - "配电箱" -> { - if (!detectResults.isContains(16)) { - warnings = "配电箱无警示标识" - alarmCode = "DistributionBoxHasNoWarningSign" - } - - if (!detectResults.isContains(36)) { - warnings = "配电箱内无电路图" - alarmCode = "DistributionBoxHasNoCircuitDiagram" - } - - if (detectResults.isContains(35)) { - warnings = "配电箱内部电线裸露" - alarmCode = "" - } - - if (detectResults.isContains(41)) { - warnings = "配电箱有跨电线" - alarmCode = "DistributionBoxHasNoJumperWire" - } - - if (segmentationResults.isNotEmpty()) { - warnings = "配电箱内电线杂乱" - alarmCode = "DistributionBoxHasTangledWire" - } - } - - "有限空间作业" -> { - if (detectResults.isContains(3)) { - if (!detectResults.isContains(15)) { - warnings = "未佩戴安全帽" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" - } - - if (!detectResults.isContains(20)) { - warnings = "未着工服" - alarmCode = "ConfinedSpaceHasNoWorkerClothes" - } - - if (!detectResults.isContains(15) && !detectResults.isContains(17)) { - warnings = "没有佩戴安全带、安全绳" - alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" - } - } - - if (!detectResults.isContains(9)) { - warnings = "未发现呼吸防护设备" - alarmCode = "ConfinedSpaceHasNoWorkerMask" - } - - if (!detectResults.isContains(42) && !detectResults.isContains(37)) { - warnings = "未发现路锥、警戒线" - alarmCode = "ConfinedSpaceHasNoEnclosure" - } - - if (!detectResults.isContains(14)) { - warnings = "未发现安全告知牌" - alarmCode = "ConfinedSpaceHasNoWarningSign" - } - - if (!detectResults.isContains(51)) { - warnings = "未发现通风设备" - alarmCode = "ConfinedSpaceHasNoAirSupply" - } - - if (!detectResults.isContains(33)) { - warnings = "未发现井下照明设备" - alarmCode = "ConfinedSpaceHasNoLighting" - } - - if (!detectResults.isContains(18)) { - warnings = "未发现井下对讲设备" - alarmCode = "ConfinedSpaceHasNoIntercom" - } - - if (!detectResults.isContains(0)) { - warnings = "未发现施工三脚架" - alarmCode = "ConfinedSpaceHasNoTripod" - } - - if (!detectResults.isContains(25)) { - warnings = "未发现气体检测仪" - alarmCode = "ConfinedSpaceHasNoGasDetector" - } - } - - "非居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - - "居民用户" -> { - if (segmentationResults.isContains(4)) { - warnings = "腐蚀、锈蚀" - alarmCode = "NonResidentUserHasPipelineRust" - } - - if (segmentationResults.isContains(0)) { - warnings = "软管不可恢复的弯折拉伸" - alarmCode = "NonResidentUserHasHoseFlexure" - } - - if (detectResults.isContains(49)) { - warnings = "非专用软管" - alarmCode = "NonResidentUserHasNonDedicatedHose" - } - - if (detectResults.isContains(1)) { - warnings = "软管有接头或三通" - alarmCode = "NonResidentUserHasNoHoseJoint" - } - - if (!detectResults.isContains(4)) { - warnings = "未发现切断阀" - alarmCode = "NonResidentUserHasNoShutoffValve" - } - - if (detectResults.isContains(34)) { - warnings = "未发现熄火保护装置" - alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" - } - - if (detectResults.isContains(22)) { - warnings = "未发现燃气泄漏报警装置" - alarmCode = "NonResidentUserHasNoAlarmDevice" - } - } - } - - if (warnings == "") { - return - } - - //显示弹框 - val message = weakReferenceHandler.obtainMessage() - message.what = 2024082902 - message.obj = warnings - weakReferenceHandler.sendMessage(message) + val segmentationResults = ArrayList() + segmentationOutput.forEach { + segmentationResults.add(it.convert2YoloResult(this)) } + + val detectResults = ArrayList() + detectOutput.forEach { + detectResults.add(it.convert2YoloResult(this)) + } + + binding.detectView.updateTargetPosition(segmentationResults, detectResults) + + lifecycleScope.launch(Dispatchers.Main) { + detectResults.forEach { + val label = LocaleConstant.CLASS_NAMES_ARRAY[it.type] + if (label.isInScene(RuntimeCache.sceneName)) { + targetSet.add(label) + } + } + } + + //只显示隐患 + var warnings = "" + when (RuntimeCache.sceneName) { + "配电箱" -> { + if (!detectResults.isContains(16)) { + warnings = "配电箱无警示标识" + alarmCode = "DistributionBoxHasNoWarningSign" + } + + if (!detectResults.isContains(36)) { + warnings = "配电箱内无电路图" + alarmCode = "DistributionBoxHasNoCircuitDiagram" + } + + if (detectResults.isContains(35)) { + warnings = "配电箱内部电线裸露" + alarmCode = "" + } + + if (detectResults.isContains(41)) { + warnings = "配电箱有跨电线" + alarmCode = "DistributionBoxHasNoJumperWire" + } + + if (segmentationResults.isNotEmpty()) { + warnings = "配电箱内电线杂乱" + alarmCode = "DistributionBoxHasTangledWire" + } + } + + "有限空间作业" -> { + if (detectResults.isContains(3)) { + if (!detectResults.isContains(15)) { + warnings = "未佩戴安全帽" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyHat" + } + + if (!detectResults.isContains(20)) { + warnings = "未着工服" + alarmCode = "ConfinedSpaceHasNoWorkerClothes" + } + + if (!detectResults.isContains(15) && !detectResults.isContains(17)) { + warnings = "没有佩戴安全带、安全绳" + alarmCode = "ConfinedSpaceHasNoWorkerSafelyLine" + } + } + + if (!detectResults.isContains(9)) { + warnings = "未发现呼吸防护设备" + alarmCode = "ConfinedSpaceHasNoWorkerMask" + } + + if (!detectResults.isContains(42) && !detectResults.isContains(37)) { + warnings = "未发现路锥、警戒线" + alarmCode = "ConfinedSpaceHasNoEnclosure" + } + + if (!detectResults.isContains(14)) { + warnings = "未发现安全告知牌" + alarmCode = "ConfinedSpaceHasNoWarningSign" + } + + if (!detectResults.isContains(51)) { + warnings = "未发现通风设备" + alarmCode = "ConfinedSpaceHasNoAirSupply" + } + + if (!detectResults.isContains(33)) { + warnings = "未发现井下照明设备" + alarmCode = "ConfinedSpaceHasNoLighting" + } + + if (!detectResults.isContains(18)) { + warnings = "未发现井下对讲设备" + alarmCode = "ConfinedSpaceHasNoIntercom" + } + + if (!detectResults.isContains(0)) { + warnings = "未发现施工三脚架" + alarmCode = "ConfinedSpaceHasNoTripod" + } + + if (!detectResults.isContains(25)) { + warnings = "未发现气体检测仪" + alarmCode = "ConfinedSpaceHasNoGasDetector" + } + } + + "非居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + + "居民用户" -> { + if (segmentationResults.isContains(4)) { + warnings = "腐蚀、锈蚀" + alarmCode = "NonResidentUserHasPipelineRust" + } + + if (segmentationResults.isContains(0)) { + warnings = "软管不可恢复的弯折拉伸" + alarmCode = "NonResidentUserHasHoseFlexure" + } + + if (detectResults.isContains(49)) { + warnings = "非专用软管" + alarmCode = "NonResidentUserHasNonDedicatedHose" + } + + if (detectResults.isContains(1)) { + warnings = "软管有接头或三通" + alarmCode = "NonResidentUserHasNoHoseJoint" + } + + if (!detectResults.isContains(4)) { + warnings = "未发现切断阀" + alarmCode = "NonResidentUserHasNoShutoffValve" + } + + if (detectResults.isContains(34)) { + warnings = "未发现熄火保护装置" + alarmCode = "NonResidentUserHasNoStoveFlameoutProtection" + } + + if (detectResults.isContains(22)) { + warnings = "未发现燃气泄漏报警装置" + alarmCode = "NonResidentUserHasNoAlarmDevice" + } + } + } + + if (warnings == "") { + return + } + + //显示弹框 + val message = weakReferenceHandler.obtainMessage() + message.what = 2024082902 + message.obj = warnings + weakReferenceHandler.sendMessage(message) } override fun onDetect(output: ArrayList) { @@ -539,7 +535,9 @@ weakReferenceHandler.sendMessage(message) } - override fun onFreeDetect(output: ArrayList) { + override fun onFreeDetect( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { } diff --git a/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt b/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt index f4e2922..9b43826 100644 --- a/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt +++ b/app/src/main/java/com/casic/br/ar/app/view/FreeModeActivity.kt @@ -21,6 +21,7 @@ class FreeModeActivity : KotlinBaseActivity(), SurfaceHolder.Callback, INativeCallback { + private val kTag = "FreeModeActivity" private val yolov8ncnn by lazy { Yolov8ncnn() } override fun initEvent() { @@ -33,10 +34,7 @@ OpenCVNativeLoader().init() //加载算法模型 - yolov8ncnn.loadModel( - assets, 1, useGpu = false, - useClassify = false, useSegmentation = false, useDetect = true - ) + yolov8ncnn.freeModeLoadModel(assets, intArrayOf(0, 1), false) binding.surfaceView.holder.setFormat(PixelFormat.RGBA_8888) binding.surfaceView.holder.addCallback(this) @@ -78,13 +76,20 @@ } - override fun onFreeDetect(output: ArrayList) { - //转成泛型集合 - val results = ArrayList() - output.forEach { - results.add(it.convert2YoloResult(this)) + override fun onFreeDetect( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + val segmentationResults = ArrayList() + segmentationOutput.forEach { + segmentationResults.add(it.convert2YoloResult(this)) } - binding.detectView.updateTargetPosition(results) + + val detectResults = ArrayList() + detectOutput.forEach { + detectResults.add(it.convert2YoloResult(this)) + } + + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun surfaceCreated(holder: SurfaceHolder) {} diff --git a/app/src/main/java/com/casic/br/ar/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/ar/app/widgets/YoloTargetDetectView.kt index f551611..f9fb342 100644 --- a/app/src/main/java/com/casic/br/ar/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/ar/app/widgets/YoloTargetDetectView.kt @@ -22,7 +22,6 @@ private val borderPaint by lazy { Paint() } private val textRect by lazy { Rect() } private val rect by lazy { Rect() } - private var freeDetectResults: MutableList = ArrayList() private var detectResults: MutableList = ArrayList() private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 @@ -45,17 +44,6 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(freeDetectResults: MutableList) { - this.freeDetectResults = freeDetectResults - postInvalidate() - } - - fun updateTargetPosition() { - this.segmentationResults.clear() - this.detectResults.clear() - postInvalidate() - } - fun updateTargetPosition( segmentationResults: MutableList, detectResults: MutableList ) { @@ -66,10 +54,6 @@ override fun onDraw(canvas: Canvas) { super.onDraw(canvas) - freeDetectResults.forEach { - drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) - } - detectResults.forEach { drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) }