diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt deleted file mode 100644 index c2dcd6d..0000000 --- a/sdk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.6) - -project(opencv_jni_shared) - -# dummy target to bring libc++_shared.so into packages -add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt deleted file mode 100644 index c2dcd6d..0000000 --- a/sdk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.6) - -project(opencv_jni_shared) - -# dummy target to bring libc++_shared.so into packages -add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/sdk/build.gradle b/sdk/build.gradle index 7784f62..f0ec9db 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -51,6 +51,7 @@ prefabPublishing true buildConfig true } + prefab { opencv_jni_shared { headers "native/jni/include" @@ -69,7 +70,7 @@ externalNativeBuild { cmake { - path (project.projectDir.toString() + '/CMakeLists.txt') + path (project.projectDir.toString() + '/libcxx_helper/CMakeLists.txt') } } } diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt deleted file mode 100644 index c2dcd6d..0000000 --- a/sdk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.6) - -project(opencv_jni_shared) - -# dummy target to bring libc++_shared.so into packages -add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/sdk/build.gradle b/sdk/build.gradle index 7784f62..f0ec9db 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -51,6 +51,7 @@ prefabPublishing true buildConfig true } + prefab { opencv_jni_shared { headers "native/jni/include" @@ -69,7 +70,7 @@ externalNativeBuild { cmake { - path (project.projectDir.toString() + '/CMakeLists.txt') + path (project.projectDir.toString() + '/libcxx_helper/CMakeLists.txt') } } } diff --git a/sdk/dummy.cpp b/sdk/dummy.cpp deleted file mode 100644 index 8b1a393..0000000 --- a/sdk/dummy.cpp +++ /dev/null @@ -1 +0,0 @@ -// empty diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt deleted file mode 100644 index c2dcd6d..0000000 --- a/sdk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.6) - -project(opencv_jni_shared) - -# dummy target to bring libc++_shared.so into packages -add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/sdk/build.gradle b/sdk/build.gradle index 7784f62..f0ec9db 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -51,6 +51,7 @@ prefabPublishing true buildConfig true } + prefab { opencv_jni_shared { headers "native/jni/include" @@ -69,7 +70,7 @@ externalNativeBuild { cmake { - path (project.projectDir.toString() + '/CMakeLists.txt') + path (project.projectDir.toString() + '/libcxx_helper/CMakeLists.txt') } } } diff --git a/sdk/dummy.cpp b/sdk/dummy.cpp deleted file mode 100644 index 8b1a393..0000000 --- a/sdk/dummy.cpp +++ /dev/null @@ -1 +0,0 @@ -// empty diff --git a/sdk/libcxx_helper/CMakeLists.txt b/sdk/libcxx_helper/CMakeLists.txt new file mode 100644 index 0000000..c2dcd6d --- /dev/null +++ b/sdk/libcxx_helper/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.6) + +project(opencv_jni_shared) + +# dummy target to bring libc++_shared.so into packages +add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.bin b/app/src/main/assets/br-detect-sim-opt-fp16.bin deleted file mode 100644 index 7ea2644..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/br-detect-sim-opt-fp16.param b/app/src/main/assets/br-detect-sim-opt-fp16.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/br-detect-sim-opt-fp16.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/assets/ji-sim-opt.bin b/app/src/main/assets/ji-sim-opt.bin deleted file mode 100644 index a2695b4..0000000 --- a/app/src/main/assets/ji-sim-opt.bin +++ /dev/null Binary files differ diff --git a/app/src/main/assets/ji-sim-opt.param b/app/src/main/assets/ji-sim-opt.param deleted file mode 100644 index 1d0e424..0000000 --- a/app/src/main/assets/ji-sim-opt.param +++ /dev/null @@ -1,183 +0,0 @@ -7767517 -181 217 -Input images 0 1 images -Convolution Conv_0 1 1 images 128 0=32 1=3 3=2 4=1 5=1 6=864 -Swish Mul_2 1 1 128 130 -Convolution Conv_3 1 1 130 131 0=64 1=3 3=2 4=1 5=1 6=18432 -Swish Mul_5 1 1 131 133 -Convolution Conv_6 1 1 133 134 0=64 1=1 5=1 6=4096 -Swish Mul_8 1 1 134 136 -Slice Split_9 1 2 136 137 138 -23300=2,32,-233 -Split splitncnn_0 1 3 138 138_splitncnn_0 138_splitncnn_1 138_splitncnn_2 -Convolution Conv_10 1 1 138_splitncnn_2 139 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_12 1 1 139 141 -Convolution Conv_13 1 1 141 142 0=32 1=3 4=1 5=1 6=9216 -Swish Mul_15 1 1 142 144 -BinaryOp Add_16 2 1 138_splitncnn_1 144 145 -Concat Concat_17 3 1 137 138_splitncnn_0 145 146 -Convolution Conv_18 1 1 146 147 0=64 1=1 5=1 6=6144 -Swish Mul_20 1 1 147 149 -Convolution Conv_21 1 1 149 150 0=128 1=3 3=2 4=1 5=1 6=73728 -Swish Mul_23 1 1 150 152 -Convolution Conv_24 1 1 152 153 0=128 1=1 5=1 6=16384 -Swish Mul_26 1 1 153 155 -Slice Split_27 1 2 155 156 157 -23300=2,64,-233 -Split splitncnn_1 1 3 157 157_splitncnn_0 157_splitncnn_1 157_splitncnn_2 -Convolution Conv_28 1 1 157_splitncnn_2 158 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_30 1 1 158 160 -Convolution Conv_31 1 1 160 161 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_33 1 1 161 163 -BinaryOp Add_34 2 1 157_splitncnn_1 163 164 -Split splitncnn_2 1 3 164 164_splitncnn_0 164_splitncnn_1 164_splitncnn_2 -Convolution Conv_35 1 1 164_splitncnn_2 165 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_37 1 1 165 167 -Convolution Conv_38 1 1 167 168 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_40 1 1 168 170 -BinaryOp Add_41 2 1 164_splitncnn_1 170 171 -Concat Concat_42 4 1 156 157_splitncnn_0 164_splitncnn_0 171 172 -Convolution Conv_43 1 1 172 173 0=128 1=1 5=1 6=32768 -Swish Mul_45 1 1 173 175 -Split splitncnn_3 1 2 175 175_splitncnn_0 175_splitncnn_1 -Convolution Conv_46 1 1 175_splitncnn_1 176 0=256 1=3 3=2 4=1 5=1 6=294912 -Swish Mul_48 1 1 176 178 -Convolution Conv_49 1 1 178 179 0=256 1=1 5=1 6=65536 -Swish Mul_51 1 1 179 181 -Slice Split_52 1 2 181 182 183 -23300=2,128,-233 -Split splitncnn_4 1 3 183 183_splitncnn_0 183_splitncnn_1 183_splitncnn_2 -Convolution Conv_53 1 1 183_splitncnn_2 184 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_55 1 1 184 186 -Convolution Conv_56 1 1 186 187 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_58 1 1 187 189 -BinaryOp Add_59 2 1 183_splitncnn_1 189 190 -Split splitncnn_5 1 3 190 190_splitncnn_0 190_splitncnn_1 190_splitncnn_2 -Convolution Conv_60 1 1 190_splitncnn_2 191 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_62 1 1 191 193 -Convolution Conv_63 1 1 193 194 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_65 1 1 194 196 -BinaryOp Add_66 2 1 190_splitncnn_1 196 197 -Concat Concat_67 4 1 182 183_splitncnn_0 190_splitncnn_0 197 198 -Convolution Conv_68 1 1 198 199 0=256 1=1 5=1 6=131072 -Swish Mul_70 1 1 199 201 -Split splitncnn_6 1 2 201 201_splitncnn_0 201_splitncnn_1 -Convolution Conv_71 1 1 201_splitncnn_1 202 0=512 1=3 3=2 4=1 5=1 6=1179648 -Swish Mul_73 1 1 202 204 -Convolution Conv_74 1 1 204 205 0=512 1=1 5=1 6=262144 -Swish Mul_76 1 1 205 207 -Slice Split_77 1 2 207 208 209 -23300=2,256,-233 -Split splitncnn_7 1 3 209 209_splitncnn_0 209_splitncnn_1 209_splitncnn_2 -Convolution Conv_78 1 1 209_splitncnn_2 210 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_80 1 1 210 212 -Convolution Conv_81 1 1 212 213 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_83 1 1 213 215 -BinaryOp Add_84 2 1 209_splitncnn_1 215 216 -Concat Concat_85 3 1 208 209_splitncnn_0 216 217 -Convolution Conv_86 1 1 217 218 0=512 1=1 5=1 6=393216 -Swish Mul_88 1 1 218 220 -Convolution Conv_89 1 1 220 221 0=256 1=1 5=1 6=131072 -Swish Mul_91 1 1 221 223 -Split splitncnn_8 1 2 223 223_splitncnn_0 223_splitncnn_1 -Pooling MaxPool_92 1 1 223_splitncnn_1 224 1=5 3=2 5=1 -Split splitncnn_9 1 2 224 224_splitncnn_0 224_splitncnn_1 -Pooling MaxPool_93 1 1 224_splitncnn_1 225 1=5 3=2 5=1 -Split splitncnn_10 1 2 225 225_splitncnn_0 225_splitncnn_1 -Pooling MaxPool_94 1 1 225_splitncnn_1 226 1=5 3=2 5=1 -Concat Concat_95 4 1 223_splitncnn_0 224_splitncnn_0 225_splitncnn_0 226 227 -Convolution Conv_96 1 1 227 228 0=512 1=1 5=1 6=524288 -Swish Mul_98 1 1 228 230 -Split splitncnn_11 1 2 230 230_splitncnn_0 230_splitncnn_1 -Interp Resize_100 1 1 230_splitncnn_1 235 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_101 2 1 235 201_splitncnn_0 236 -Convolution Conv_102 1 1 236 237 0=256 1=1 5=1 6=196608 -Swish Mul_104 1 1 237 239 -Slice Split_105 1 2 239 240 241 -23300=2,128,-233 -Split splitncnn_12 1 2 241 241_splitncnn_0 241_splitncnn_1 -Convolution Conv_106 1 1 241_splitncnn_1 242 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_108 1 1 242 244 -Convolution Conv_109 1 1 244 245 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_111 1 1 245 247 -Concat Concat_112 3 1 240 241_splitncnn_0 247 248 -Convolution Conv_113 1 1 248 249 0=256 1=1 5=1 6=98304 -Swish Mul_115 1 1 249 251 -Split splitncnn_13 1 2 251 251_splitncnn_0 251_splitncnn_1 -Interp Resize_117 1 1 251_splitncnn_1 256 0=1 1=2.000000e+00 2=2.000000e+00 -Concat Concat_118 2 1 256 175_splitncnn_0 257 -Convolution Conv_119 1 1 257 258 0=128 1=1 5=1 6=49152 -Swish Mul_121 1 1 258 260 -Slice Split_122 1 2 260 261 262 -23300=2,64,-233 -Split splitncnn_14 1 2 262 262_splitncnn_0 262_splitncnn_1 -Convolution Conv_123 1 1 262_splitncnn_1 263 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_125 1 1 263 265 -Convolution Conv_126 1 1 265 266 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_128 1 1 266 268 -Concat Concat_129 3 1 261 262_splitncnn_0 268 269 -Convolution Conv_130 1 1 269 270 0=128 1=1 5=1 6=24576 -Swish Mul_132 1 1 270 272 -Split splitncnn_15 1 3 272 272_splitncnn_0 272_splitncnn_1 272_splitncnn_2 -Convolution Conv_133 1 1 272_splitncnn_2 273 0=128 1=3 3=2 4=1 5=1 6=147456 -Swish Mul_135 1 1 273 275 -Concat Concat_136 2 1 275 251_splitncnn_0 276 -Convolution Conv_137 1 1 276 277 0=256 1=1 5=1 6=98304 -Swish Mul_139 1 1 277 279 -Slice Split_140 1 2 279 280 281 -23300=2,128,-233 -Split splitncnn_16 1 2 281 281_splitncnn_0 281_splitncnn_1 -Convolution Conv_141 1 1 281_splitncnn_1 282 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_143 1 1 282 284 -Convolution Conv_144 1 1 284 285 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_146 1 1 285 287 -Concat Concat_147 3 1 280 281_splitncnn_0 287 288 -Convolution Conv_148 1 1 288 289 0=256 1=1 5=1 6=98304 -Swish Mul_150 1 1 289 291 -Split splitncnn_17 1 3 291 291_splitncnn_0 291_splitncnn_1 291_splitncnn_2 -Convolution Conv_151 1 1 291_splitncnn_2 292 0=256 1=3 3=2 4=1 5=1 6=589824 -Swish Mul_153 1 1 292 294 -Concat Concat_154 2 1 294 230_splitncnn_0 295 -Convolution Conv_155 1 1 295 296 0=512 1=1 5=1 6=393216 -Swish Mul_157 1 1 296 298 -Slice Split_158 1 2 298 299 300 -23300=2,256,-233 -Split splitncnn_18 1 2 300 300_splitncnn_0 300_splitncnn_1 -Convolution Conv_159 1 1 300_splitncnn_1 301 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_161 1 1 301 303 -Convolution Conv_162 1 1 303 304 0=256 1=3 4=1 5=1 6=589824 -Swish Mul_164 1 1 304 306 -Concat Concat_165 3 1 299 300_splitncnn_0 306 307 -Convolution Conv_166 1 1 307 308 0=512 1=1 5=1 6=393216 -Swish Mul_168 1 1 308 310 -Split splitncnn_19 1 2 310 310_splitncnn_0 310_splitncnn_1 -Convolution Conv_169 1 1 272_splitncnn_1 311 0=64 1=3 4=1 5=1 6=73728 -Swish Mul_171 1 1 311 313 -Convolution Conv_172 1 1 313 314 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_174 1 1 314 316 -Convolution Conv_175 1 1 316 317 0=64 1=1 5=1 6=4096 -Convolution Conv_176 1 1 272_splitncnn_0 318 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_178 1 1 318 320 -Convolution Conv_179 1 1 320 321 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_181 1 1 321 323 -Convolution Conv_182 1 1 323 324 0=5 1=1 5=1 6=640 -Concat Concat_183 2 1 317 324 325 -Convolution Conv_184 1 1 291_splitncnn_1 326 0=64 1=3 4=1 5=1 6=147456 -Swish Mul_186 1 1 326 328 -Convolution Conv_187 1 1 328 329 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_189 1 1 329 331 -Convolution Conv_190 1 1 331 332 0=64 1=1 5=1 6=4096 -Convolution Conv_191 1 1 291_splitncnn_0 333 0=128 1=3 4=1 5=1 6=294912 -Swish Mul_193 1 1 333 335 -Convolution Conv_194 1 1 335 336 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_196 1 1 336 338 -Convolution Conv_197 1 1 338 339 0=5 1=1 5=1 6=640 -Concat Concat_198 2 1 332 339 340 -Convolution Conv_199 1 1 310_splitncnn_1 341 0=64 1=3 4=1 5=1 6=294912 -Swish Mul_201 1 1 341 343 -Convolution Conv_202 1 1 343 344 0=64 1=3 4=1 5=1 6=36864 -Swish Mul_204 1 1 344 346 -Convolution Conv_205 1 1 346 347 0=64 1=1 5=1 6=4096 -Convolution Conv_206 1 1 310_splitncnn_0 348 0=128 1=3 4=1 5=1 6=589824 -Swish Mul_208 1 1 348 350 -Convolution Conv_209 1 1 350 351 0=128 1=3 4=1 5=1 6=147456 -Swish Mul_211 1 1 351 353 -Convolution Conv_212 1 1 353 354 0=5 1=1 5=1 6=640 -Concat Concat_213 2 1 347 354 355 -Reshape Reshape_219 1 1 325 365 0=-1 1=69 -Reshape Reshape_222 1 1 340 372 0=-1 1=69 -Reshape Reshape_225 1 1 355 379 0=-1 1=69 -Concat Concat_226 3 1 365 372 379 380 0=1 -Permute Transpose_227 1 1 380 output 0=1 diff --git a/app/src/main/cpp/yolo.cpp b/app/src/main/cpp/yolo.cpp index 62e2448..2871f8d 100644 --- a/app/src/main/cpp/yolo.cpp +++ b/app/src/main/cpp/yolo.cpp @@ -354,30 +354,48 @@ } int -Yolo::load(const char *model_type, int target_size, const float *mean_values, - const float *norm_values, bool use_gpu) { - -} - -int Yolo::load(AAssetManager *mgr, const char *model_type, int _target_size, const float *_mean_values, - const float *_norm_values, bool use_gpu) { - yolo.clear(); + const float *_norm_values, bool use_classify, bool use_segmentation, bool use_detect, + bool use_gpu) { + if (use_classify) { + yolo_c.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c.clear()"); + } + if (use_segmentation) { + yolo_s.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s.clear()"); + } + if (use_detect) { + yolo_d.clear(); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d.clear()"); + } + blob_pool_allocator.clear(); workspace_pool_allocator.clear(); ncnn::set_cpu_powersave(2); ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); - yolo.opt = ncnn::Option(); + yolo_c.opt = ncnn::Option(); + yolo_s.opt = ncnn::Option(); + yolo_d.opt = ncnn::Option(); #if NCNN_VULKAN - yolo.opt.use_vulkan_compute = use_gpu; + yolo_c.opt.use_vulkan_compute = use_gpu; + yolo_s.opt.use_vulkan_compute = use_gpu; + yolo_d.opt.use_vulkan_compute = use_gpu; #endif + yolo_c.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_c.opt.blob_allocator = &blob_pool_allocator; + yolo_c.opt.workspace_allocator = &workspace_pool_allocator; - yolo.opt.num_threads = ncnn::get_big_cpu_count(); - yolo.opt.blob_allocator = &blob_pool_allocator; - yolo.opt.workspace_allocator = &workspace_pool_allocator; + yolo_s.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_s.opt.blob_allocator = &blob_pool_allocator; + yolo_s.opt.workspace_allocator = &workspace_pool_allocator; + + yolo_d.opt.num_threads = ncnn::get_big_cpu_count(); + yolo_d.opt.blob_allocator = &blob_pool_allocator; + yolo_d.opt.workspace_allocator = &workspace_pool_allocator; char param_path[256]; char model_path[256]; @@ -388,8 +406,24 @@ __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "param_path %s", param_path); __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "model_path %s", model_path); - yolo.load_param(mgr, param_path); - yolo.load_model(mgr, model_path); + if (use_classify) { + yolo_c.load_param(mgr, param_path); + yolo_c.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_c load model %s", model_path); + } + if (use_segmentation) { + yolo_s.load_param(mgr, param_path); + yolo_s.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_s load model %s", model_path); + } + if (use_detect) { + yolo_d.load_param(mgr, param_path); + yolo_d.load_model(mgr, model_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load param %s", param_path); + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "yolo_d load model %s", model_path); + } target_size = _target_size; mean_values[0] = _mean_values[0]; @@ -416,7 +450,7 @@ } int Yolo::classify(const cv::Mat &rgb) { - if (state == 0) { + if (j_state == 0) { static const float scale_values[3] = {0.017f, 0.017f, 0.017f}; int width = rgb.cols; @@ -428,7 +462,7 @@ std::vector cls_scores; { in.substract_mean_normalize(mean_values, scale_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_c.create_extractor(); ex.input("images", in); ncnn::Mat out; @@ -457,9 +491,9 @@ return 0; } -int Yolo::partition(const cv::Mat &rgb, std::vector &objects, float prob_threshold, - float nms_threshold) { - if (state == 1) { +int Yolo::segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold, + float nms_threshold) { + if (j_state == 1) { int width = rgb.cols; int height = rgb.rows; @@ -481,135 +515,203 @@ height, w, h); // pad to target_size rectangle - int wpad = (w + 31) / 32 * 32 - w; - int hpad = (h + 31) / 32 * 32 - h; + int w_pad = (w + 31) / 32 * 32 - w; + int h_pad = (h + 31) / 32 * 32 - h; ncnn::Mat in_pad; - ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, - ncnn::BORDER_CONSTANT, 0.f); + ncnn::copy_make_border(in, in_pad, h_pad / 2, h_pad - h_pad / 2, w_pad / 2, + w_pad - w_pad / 2, ncnn::BORDER_CONSTANT, 0.f); - const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; - in_pad.substract_mean_normalize(0, norm_vals); + in_pad.substract_mean_normalize(0, norm_values); + JNIEnv *env; + javaVM->AttachCurrentThread(&env, nullptr); + jclass callback_clazz = env->GetObjectClass(j_callback); + jmethodID j_method_id = env->GetMethodID( + callback_clazz, "onSegmentation", "(Ljava/util/ArrayList;Ljava/util/ArrayList;)V" + ); - ncnn::Extractor ex = yolo.create_extractor(); - ex.input("images", in_pad); + //获取ArrayList类 + jclass list_clazz = env->FindClass("java/util/ArrayList"); + jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); + jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); + //初始化ArrayList对象 + jobject segment_array_obj = env->NewObject(list_clazz, arraylist_init); + jobject detect_array_obj = env->NewObject(list_clazz, arraylist_init); - ncnn::Mat out; - ex.extract("output", out); + //分割 + { + ncnn::Extractor ex = yolo_s.create_extractor(); + ex.input("images", in_pad); - ncnn::Mat mask_proto; - ex.extract("seg", mask_proto); + ncnn::Mat out; + ex.extract("output", out); - std::vector strides = {8, 16, 32}; - std::vector grid_strides; - generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + ncnn::Mat mask_proto; + ex.extract("seg", mask_proto); - std::vector proposals; - std::vector objects8; - generate_proposals(grid_strides, out, prob_threshold, objects8, 6); + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); - proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + std::vector proposals; + std::vector objects8; + generate_proposals(grid_strides, out, prob_threshold, objects8, 6); - // sort all proposals by score from highest to lowest - qsort_descent_inplace(proposals); + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); - // apply nms with nms_threshold - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); - int count = picked.size(); + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); - ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); - for (int i = 0; i < count; i++) { - float *mask_feat_ptr = mask_feat.row(i); - std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), - sizeof(float) * proposals[picked[i]].mask_feat.size()); + int count = picked.size(); + + ncnn::Mat mask_feat = ncnn::Mat(32, count, sizeof(float)); + for (int i = 0; i < count; i++) { + float *mask_feat_ptr = mask_feat.row(i); + std::memcpy(mask_feat_ptr, proposals[picked[i]].mask_feat.data(), + sizeof(float) * proposals[picked[i]].mask_feat.size()); + } + + ncnn::Mat mask_pred_result; + decode_mask(mask_feat, width, height, mask_proto, in_pad, w_pad, h_pad, + mask_pred_result); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); + cv::Mat mask = cv::Mat(height, width, CV_32FC1, + (float *) mask_pred_result.channel(i)); + mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + } + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(segment_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - ncnn::Mat mask_pred_result; - decode_mask(mask_feat, width, height, mask_proto, in_pad, wpad, hpad, mask_pred_result); + //检测 + { + ncnn::Extractor ex = yolo_d.create_extractor(); - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; + ex.input("images", in_pad); - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + std::vector proposals; - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + ncnn::Mat out; + ex.extract("output", out); - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; + std::vector strides = {8, 16, 32}; // might have stride=64 + std::vector grid_strides; + generate_grids_and_stride(in_pad.w, in_pad.h, strides, grid_strides); + generate_proposals(grid_strides, out, 0.4f, proposals, 43); - objects[i].mask = cv::Mat::zeros(height, width, CV_32FC1); - cv::Mat mask = cv::Mat(height, width, CV_32FC1, (float *) mask_pred_result.channel(i)); - mask(objects[i].rect).copyTo(objects[i].mask(objects[i].rect)); + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, 0.5f); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (w_pad / 2)) / scale; + float y0 = (objects[i].rect.y - (h_pad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (w_pad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (h_pad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); + y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); + x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); + y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + // sort objects by area + struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + for (const auto &item: objects) { + float array[4]; + array[0] = item.rect.x; + array[1] = item.rect.y; + array[2] = item.rect.width; + array[3] = item.rect.height; + + char text[256]; + sprintf( + text, + "%d %f %f %f %f %.1f%%", + item.label, + array[0], array[1], array[2], array[3], + item.prob * 100 + ); + + //add + env->CallBooleanMethod(detect_array_obj, arraylist_add, env->NewStringUTF(text)); + } } - /** - * 回调给Java/Kotlin层 - * */ -// JNIEnv *env; -// javaVM->AttachCurrentThread(&env, nullptr); -// jclass callback_clazz = env->GetObjectClass(j_callback); -// jclass output_clazz = env->GetObjectClass(j_output); -// -// jmethodID j_method_id = env->GetMethodID( -// callback_clazz, "onPartition", "(Ljava/util/ArrayList;)V" -// ); -// -// //获取ArrayList类 -// jclass list_clazz = env->FindClass("java/util/ArrayList"); -// jmethodID arraylist_init = env->GetMethodID(list_clazz, "", "()V"); -// jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); -// //初始化ArrayList对象 -// jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); -// -// for (auto item: objects) { -// jfieldID type = env->GetFieldID(output_clazz, "type", "I"); -// env->SetIntField(j_output, type, item.label); -// -// jfieldID position = env->GetFieldID(output_clazz, "position", "[F"); -// float array[4]; -// array[0] = item.rect.x; -// array[1] = item.rect.y; -// array[2] = item.rect.width; -// array[3] = item.rect.height; -// jfloatArray rectArray = env->NewFloatArray(4); -// env->SetFloatArrayRegion(rectArray, 0, 4, array); -// env->SetObjectField(j_output, position, rectArray); -// -// jfieldID prob = env->GetFieldID(output_clazz, "prob", "F"); -// env->SetFloatField(j_output, prob, item.prob); -// -// //add -// env->CallBooleanMethod(arraylist_obj, arraylist_add, j_output); -// } -// //回调 -// env->CallVoidMethod(j_callback, j_method_id, arraylist_obj); -// -// /** -// * Mat数据。 -// * */ -// auto *res = (cv::Mat *) j_mat_addr; -// res->create(rgb.rows, rgb.cols, rgb.type()); -// memcpy(res->data, rgb.data, rgb.rows * rgb.step); + //分割和检测结果统一回调 + env->CallVoidMethod(j_callback, j_method_id, segment_array_obj, detect_array_obj); } return 0; } int Yolo::detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold, float nms_threshold) { - if (state == 2) { + if (j_state == 2) { int width = rgb.cols; int height = rgb.rows; @@ -643,7 +745,7 @@ in_pad.substract_mean_normalize(0, norm_values); - ncnn::Extractor ex = yolo.create_extractor(); + ncnn::Extractor ex = yolo_d.create_extractor(); ex.input("images", in_pad); @@ -713,10 +815,7 @@ jmethodID arraylist_add = env->GetMethodID(list_clazz, "add", "(Ljava/lang/Object;)Z"); //初始化ArrayList对象 jobject arraylist_obj = env->NewObject(list_clazz, arraylist_init); - - for (int i = 0; i < count; i++) { - auto item = objects[i]; - + for (const auto &item: objects) { float array[4]; array[0] = item.rect.x; array[1] = item.rect.y; diff --git a/app/src/main/cpp/yolo.h b/app/src/main/cpp/yolo.h index f9dcb87..b047291 100644 --- a/app/src/main/cpp/yolo.h +++ b/app/src/main/cpp/yolo.h @@ -44,19 +44,16 @@ * 2 - 检测
* 3 - 绘制
* */ - int state = 0; - - int load(const char *model_type, - int target_size, - const float *mean_values, - const float *norm_values, - bool use_gpu = false); + int j_state = 0; int load(AAssetManager *mgr, const char *model_type, int target_size, const float *mean_values, const float *norm_values, + bool use_classify, + bool use_segmentation, + bool use_detect, bool use_gpu = false); void initNativeCallback(JavaVM *vm, jlong nativeObjAddr, jobject pJobject); @@ -69,23 +66,24 @@ /** * 分割 * */ - int partition(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, - float nms_threshold = 0.5f); + int segmentation(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, + float nms_threshold = 0.5f); /** * 检测 * */ - int detect(const cv::Mat &rgb, - std::vector &objects, - float prob_threshold = 0.4f, + int detect(const cv::Mat &rgb, std::vector &objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f); int draw(cv::Mat &rgb, const std::vector &objects); private: - ncnn::Net yolo; + //分类 + ncnn::Net yolo_c; + //分割 + ncnn::Net yolo_s; + //检测 + ncnn::Net yolo_d; int target_size; float mean_values[3]; float norm_values[3]; diff --git a/app/src/main/cpp/yolov8ncnn.cpp b/app/src/main/cpp/yolov8ncnn.cpp index 8cdd4a6..38cdadc 100644 --- a/app/src/main/cpp/yolov8ncnn.cpp +++ b/app/src/main/cpp/yolov8ncnn.cpp @@ -128,7 +128,7 @@ std::vector objects; //分割 - g_yolo->partition(rgb, objects); + g_yolo->segmentation(rgb, objects); //检测 g_yolo->detect(rgb, objects); @@ -140,6 +140,20 @@ static MyNdkCamera *g_camera = nullptr; +//分割、分类、检测 +const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; +const int target_sizes[] = {320, 320, 320}; +const float mean_values[][3] = { + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f}, + {103.53f, 116.28f, 123.675f} +}; +const float norm_values[][3] = { + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f}, + {1 / 255.f, 1 / 255.f, 1 / 255.f} +}; + extern "C" { JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); @@ -167,33 +181,18 @@ JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_loadModel(JNIEnv *env, jobject thiz, jobject assetManager, - jint model_id, jint processor) { - if (model_id < 0 || model_id > 6 || processor < 0 || processor > 1) { + jint model_id, jboolean use_gpu, + jboolean use_classify, + jboolean use_segmentation, + jboolean use_detect) { + if (model_id < 0 || model_id > 2) { return JNI_FALSE; } AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); - //分割、分类、检测 - const char *model_types[] = {"best-sim-opt-fp16", "model.ncnn", "yolov8s-detect-sim-opt-fp16"}; - - const int target_sizes[] = {320, 320, 320}; - - const float mean_values[][3] = { - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f}, - {103.53f, 116.28f, 123.675f} - }; - - const float norm_values[][3] = { - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f}, - {1 / 255.f, 1 / 255.f, 1 / 255.f} - }; - const char *model_type = model_types[(int) model_id]; int target_size = target_sizes[(int) model_id]; - bool use_gpu = (int) processor == 1; // reload { @@ -206,12 +205,27 @@ } else { if (!g_yolo) g_yolo = new Yolo; + int state; + if (use_classify) { + state = 0; + } + if (use_segmentation) { + state = 1; + } + if (use_detect) { + state = 2; + } + g_yolo->j_state = state; g_yolo->load( mgr, model_type, target_size, mean_values[(int) model_id], - norm_values[(int) model_id], use_gpu + norm_values[(int) model_id], + use_classify, + use_segmentation, + use_detect, + use_gpu ); } } @@ -220,6 +234,62 @@ } JNIEXPORT jboolean JNICALL +Java_com_casic_br_app_external_Yolov8ncnn_loadMultiModel(JNIEnv *env, jobject thiz, + jobject assetManager, + jintArray ids, jboolean use_gpu) { + AAssetManager *mgr = AAssetManager_fromJava(env, assetManager); + + jint *intArray = env->GetIntArrayElements(ids, nullptr); + jsize len = env->GetArrayLength(ids); + for (int i = 0; i < len; i++) { + int *id = intArray + i; + const char *model_type = model_types[*id]; + int target_size = target_sizes[*id]; + + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) { + // no gpu + delete g_yolo; + g_yolo = nullptr; + } else { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->j_state = 1; + if (*id == 0) { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + true, + false, + use_gpu + ); + } else { + g_yolo->load( + mgr, + model_type, + target_size, + mean_values[*id], + norm_values[*id], + false, + false, + true, + use_gpu + ); + } + } + } + } + + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL Java_com_casic_br_app_external_Yolov8ncnn_openCamera(JNIEnv *env, jobject thiz, jint facing) { if (facing < 0 || facing > 1) return JNI_FALSE; @@ -248,23 +318,14 @@ jobject native_callback) { ANativeWindow *win = ANativeWindow_fromSurface(env, surface); - __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); - g_camera->set_window(win); g_yolo->initNativeCallback(javaVM, nativeObjAddr, native_callback); return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_casic_br_app_external_Yolov8ncnn_updateYoloState(JNIEnv *env, jobject thiz, - jint yolo_state) { - g_yolo->state = yolo_state; - return JNI_TRUE; -} - JNIEXPORT jint JNICALL Java_com_casic_br_app_external_Yolov8ncnn_getYoloCurrentState(JNIEnv *env, jobject thiz) { - return g_yolo->state; + return g_yolo->j_state; } -} +} \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt index a9a9a36..c2abd62 100644 --- a/app/src/main/java/com/casic/br/app/external/INativeCallback.kt +++ b/app/src/main/java/com/casic/br/app/external/INativeCallback.kt @@ -9,7 +9,7 @@ /** * 分割 */ - fun onPartition(output: ArrayList) + fun onSegmentation(segmentationOutput: ArrayList, detectOutput: ArrayList) /** * 检测 diff --git a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt index ebeaf20..d615b31 100644 --- a/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt +++ b/app/src/main/java/com/casic/br/app/external/YoloStateConst.kt @@ -12,7 +12,7 @@ * 3 - 绘制 * */ const val CLASSIFY = 0 - const val PARTITION = 1 + const val SEGMENTATION = 1 const val DETECT = 2 const val DRAW = 3 } diff --git a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt index dd99b02..ba7bca8 100644 --- a/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt +++ b/app/src/main/java/com/casic/br/app/external/Yolov8ncnn.kt @@ -13,10 +13,23 @@ /** * @param mgr 手机内存资源管理器 - * @param model_id 模型ID - * @param processor 是否使用GPU 1-是，0-否 + * @param modelId 模型ID + * @param useGpu 是否使用GPU + * @param useClassify 是否使用分类模型 + * @param useSegmentation 是否使用分割模型 + * @param useDetect 是否使用检测模型 * */ - external fun loadModel(mgr: AssetManager, model_id: Int, processor: Int): Boolean + external fun loadModel( + mgr: AssetManager, modelId: Int, useGpu: Boolean, useClassify: Boolean, + useSegmentation: Boolean, useDetect: Boolean + ): Boolean + + /** + * @param mgr 手机内存资源管理器 + * @param ids 多模型ID数组 + * @param useGpu 是否使用GPU + * */ + external fun loadMultiModel(mgr: AssetManager, ids: IntArray, useGpu: Boolean): Boolean /** * @param facing 相机 0-前置镜头，1-后置镜头 @@ -29,7 +42,5 @@ surface: Surface, nativeObjAddr: Long, callBack: INativeCallback ): Boolean - external fun updateYoloState(@YoloStateConst yoloState: Int): Boolean - external fun getYoloCurrentState(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt index 62d23b2..ce34c17 100644 --- a/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt +++ b/app/src/main/java/com/casic/br/app/utils/LocaleConstant.kt @@ -51,6 +51,8 @@ "长柄阀门", "阀门", "风管" ) + val SEGMENTATION_ARRAY = arrayOf("弯折", "断裂", "烧焦", "磨损", "铁锈", "龟裂") + //配电箱 val DISTRIBUTION_BOX_SCENE_ARRAY = arrayListOf( "安全标识", "电路图", "电线暴露", "跨电线", "配电箱", "贴纸", "开关" diff --git a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt index cc1f13e..8c0c646 100644 --- a/app/src/main/java/com/casic/br/app/view/LoginActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/LoginActivity.kt @@ -2,11 +2,9 @@ import android.os.Bundle import androidx.lifecycle.ViewModelProvider -import androidx.lifecycle.lifecycleScope import com.amap.api.maps.MapsInitializer import com.casic.br.app.R import com.casic.br.app.databinding.ActivityLoginBinding -import com.casic.br.app.external.YoloStateConst import com.casic.br.app.external.Yolov8ncnn import com.casic.br.app.utils.AuthenticationHelper import com.casic.br.app.utils.LocaleConstant @@ -21,8 +19,6 @@ import com.pengxh.kt.lite.utils.LoadState import com.pengxh.kt.lite.utils.LoadingDialogHub import com.pengxh.kt.lite.utils.SaveKeyValues -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.opencv.android.OpenCVLoader import pub.devrel.easypermissions.EasyPermissions @@ -48,12 +44,6 @@ binding.userNameView.setText(SaveKeyValues.getValue(LocaleConstant.ACCOUNT, "") as String) binding.passwordView.setText(SaveKeyValues.getValue(LocaleConstant.PASSWORD, "") as String) - //协程预加载算法模型 - lifecycleScope.launch(Dispatchers.IO) { - yolov8ncnn.loadModel(assets, 1, 0) - yolov8ncnn.updateYoloState(YoloStateConst.CLASSIFY) - } - //初始化OpenCV OpenCVLoader.initLocal() diff --git a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt index fc8dccf..8002bd3 100644 --- a/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt +++ b/app/src/main/java/com/casic/br/app/view/StartCheckByYoloActivity.kt @@ -90,7 +90,7 @@ val inspectionDate = currentTimeMillis.timestampToDate() val startTime = currentTimeMillis.timestampToTime() inspectionViewModel.startInspection( - context, inspectionUser, inspectionAddress, inspectionDate, startTime + this, inspectionUser, inspectionAddress, inspectionDate, startTime ) inspectionViewModel.addResultModel.observe(this) { if (it.code == 200) { @@ -223,11 +223,12 @@ maxIndex = index } } + Log.d(kTag, "maxIndex: $maxIndex") try { lifecycleScope.launch(Dispatchers.Main) { isShowing = true - if (classArray[maxIndex] == classArray[2]) { + if (maxIndex == 2) { //非居 AlertControlDialog.Builder() .setContext(context) @@ -238,7 +239,7 @@ AlertControlDialog.OnDialogButtonClickListener { override fun onConfirmClick() { //需要同时调用分割和检测模型 - + yolov8ncnn.loadMultiModel(assets, intArrayOf(0, 2), false) } override fun onCancelClick() { @@ -247,9 +248,15 @@ }).build().show() } else { if (yolov8ncnn.getYoloCurrentState() != YoloStateConst.DETECT) { - //需要调用检测模型 - loadModelFromAssets(2) - yolov8ncnn.updateYoloState(YoloStateConst.DETECT) + //调用检测模型 + yolov8ncnn.loadModel( + assets, + 2, + useGpu = false, + useClassify = false, + useSegmentation = false, + useDetect = true + ) } } } @@ -258,8 +265,46 @@ } } - override fun onPartition(output: ArrayList) { + override fun onSegmentation( + segmentationOutput: ArrayList, detectOutput: ArrayList + ) { + //转成泛型集合 + val segmentationResults = ArrayList() + segmentationOutput.forEach { + val yolo = YoloResult() + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + segmentationResults.add(yolo) + } + + val detectResults = ArrayList() + detectOutput.forEach { + val yolo = YoloResult() + + val strings = it.split(" ") + yolo.type = strings.first().toInt() + + val array = FloatArray(4) + array[0] = strings[1].toFloat() + array[1] = strings[2].toFloat() + array[2] = strings[3].toFloat() + array[3] = strings[4].toFloat() + yolo.position = array + + yolo.prob = strings.last() + detectResults.add(yolo) + } + binding.detectView.updateTargetPosition(segmentationResults, detectResults) } override fun onDetect(output: ArrayList) { @@ -303,7 +348,6 @@ // } } } - Log.d(kTag, results.toJson()) binding.detectView.updateTargetPosition(results) // if (mat.width() > 0 && mat.height() > 0) { // val bitmap = Bitmap.createBitmap(mat.width(), mat.height(), Bitmap.Config.ARGB_8888) @@ -320,6 +364,16 @@ override fun onResume() { super.onResume() + //加载算法模型 + yolov8ncnn.loadModel( + assets, + 1, + useGpu = false, + useClassify = true, + useSegmentation = false, + useDetect = false + ) + yolov8ncnn.openCamera(1) } @@ -332,14 +386,4 @@ super.onDestroy() locationManager.stopLocation() } - - /** - * index对应 JNI 里面定义的数组角标 - * */ - private fun loadModelFromAssets(index: Int) { - val result = yolov8ncnn.loadModel(assets, index, 0) - if (!result) { - Log.d(kTag, "reload: yolov8ncnn loadModel failed") - } - } } \ No newline at end of file diff --git a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt index 3442717..7d2c7cc 100644 --- a/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt +++ b/app/src/main/java/com/casic/br/app/widgets/YoloTargetDetectView.kt @@ -21,7 +21,8 @@ private val backgroundPaint by lazy { Paint() } private val borderPaint by lazy { Paint() } private val rect by lazy { Rect() } - private var results = ArrayList() + private var results: MutableList = ArrayList() + private var segmentationResults: MutableList = ArrayList() private var textHeight = 0 init { @@ -42,42 +43,56 @@ borderPaint.isAntiAlias = true } - fun updateTargetPosition(results: ArrayList) { + fun updateTargetPosition(results: MutableList) { this.results = results invalidate() } + fun updateTargetPosition( + segmentationResults: MutableList, detectResults: MutableList + ) { + this.segmentationResults = segmentationResults + this.results = detectResults + postInvalidate() + } + override fun onDraw(canvas: Canvas) { super.onDraw(canvas) results.forEach { - val label = "${LocaleConstant.CLASS_NAMES_ARRAY[it.type]} ${it.prob}" - val textLength = textPaint.measureText(label) - - //文字背景 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[0].dp2px(context) + textLength).toInt() + 10, - it.position[1].dp2px(context).toInt() - textHeight - ) - canvas.drawRect(rect, backgroundPaint) - - //画文字。数值是文字左右边距，可酌情调整 - canvas.drawText( - label, - it.position[0].dp2px(context) + (textLength + 10) / 2, - it.position[1].dp2px(context) - 10, - textPaint - ) - - //画框 - rect.set( - (it.position[0].dp2px(context)).toInt(), - (it.position[1].dp2px(context)).toInt(), - (it.position[2] + it.position[0]).dp2px(context).toInt(), - (it.position[3] + it.position[1]).dp2px(context).toInt() - ) - canvas.drawRect(rect, borderPaint) + drawTarget(canvas, it, LocaleConstant.CLASS_NAMES_ARRAY[it.type]) } + + segmentationResults.forEach { + drawTarget(canvas, it, LocaleConstant.SEGMENTATION_ARRAY[it.type]) + } + } + + private fun drawTarget(canvas: Canvas, it: YoloResult, label: String) { + val textLength = textPaint.measureText(label) + //文字背景。数字仅为了纠正背景和文字以及边框对齐，因为坐标值转px时会丢失一次精度，转int会再丢失一次精度，最后会导致背景和文字以及边框无法完美对齐 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[0].dp2px(context) + textLength).toInt() + 10, + it.position[1].dp2px(context).toInt() - textHeight + ) + canvas.drawRect(rect, backgroundPaint) + + //画文字。数值是文字左右边距，可酌情调整 + canvas.drawText( + label, + it.position[0].dp2px(context) + (textLength + 10) / 2, + it.position[1].dp2px(context) - 10, + textPaint + ) + + //画框 + rect.set( + (it.position[0].dp2px(context)).toInt(), + (it.position[1].dp2px(context)).toInt(), + (it.position[2] + it.position[0]).dp2px(context).toInt(), + (it.position[3] + it.position[1]).dp2px(context).toInt() + ) + canvas.drawRect(rect, borderPaint) } } \ No newline at end of file diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt deleted file mode 100644 index c2dcd6d..0000000 --- a/sdk/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.6) - -project(opencv_jni_shared) - -# dummy target to bring libc++_shared.so into packages -add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/sdk/build.gradle b/sdk/build.gradle index 7784f62..f0ec9db 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -51,6 +51,7 @@ prefabPublishing true buildConfig true } + prefab { opencv_jni_shared { headers "native/jni/include" @@ -69,7 +70,7 @@ externalNativeBuild { cmake { - path (project.projectDir.toString() + '/CMakeLists.txt') + path (project.projectDir.toString() + '/libcxx_helper/CMakeLists.txt') } } } diff --git a/sdk/dummy.cpp b/sdk/dummy.cpp deleted file mode 100644 index 8b1a393..0000000 --- a/sdk/dummy.cpp +++ /dev/null @@ -1 +0,0 @@ -// empty diff --git a/sdk/libcxx_helper/CMakeLists.txt b/sdk/libcxx_helper/CMakeLists.txt new file mode 100644 index 0000000..c2dcd6d --- /dev/null +++ b/sdk/libcxx_helper/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.6) + +project(opencv_jni_shared) + +# dummy target to bring libc++_shared.so into packages +add_library(opencv_jni_shared STATIC dummy.cpp) diff --git a/sdk/libcxx_helper/dummy.cpp b/sdk/libcxx_helper/dummy.cpp new file mode 100644 index 0000000..8b1a393 --- /dev/null +++ b/sdk/libcxx_helper/dummy.cpp @@ -0,0 +1 @@ +// empty