import 'dart:typed_data'; import 'package:flutter/foundation.dart' show debugPrint; import 'package:tflite_flutter/tflite_flutter.dart'; import 'detection_result.dart'; import 'nms.dart'; /// YOLOv8s 端侧推理实现(对应 Kotlin TFLiteDetector)。 /// 模型输出布局(ultralytics litert 导出):[1, 4 + nc, anchors], /// cx/cy/w/h 已归一化,类别得分已过 sigmoid;按 out[dim][anchor] 索引。 /// 输入为 NCHW [1, 3, 1280, 1280](litert 导出保留 torch 布局)。 class TfliteDetector { // 输入尺寸取自模型本身(ultralytics litert 导出 NCHW [1,3,H,W],各数据集 // 训练 imgsz 可不同),默认 1280 兜底 static const int defaultInputSize = 1280; // 环颈雉鸡数据置信度普遍偏低(0.1~0.2 量级),保留低分池供运动检测提升; // 可运行时调整(设置页滑块),默认 0.10 double minScore = 0.10; static const double iouThreshold = 0.45; static const int maxDetections = 20; final Interpreter _interpreter; final List _labels; final int _numClasses; final int _numAnchors; final int inputSize; /// 模型身份(多模型并行推理区分来源;无来源为 -1/空) final int modelId; final String modelName; late final Float32List _input; /// 输出按模型形状 [1, 4+nc, anchors] 的嵌套 List 组织, /// run() 要求输出对象形状与模型完全一致(扁平 List 会被拒)。 final List>> _output; TfliteDetector._(this._interpreter, this._labels, this._numClasses, this._numAnchors, this._output, this.inputSize, this.modelId, this.modelName) { _input = Float32List(1 * inputSize * inputSize * 3); } /// 模型缺失或加载失败返回 null(App 降级为仅预览)。 /// 在后台 isolate 内调用(模型字节由主 isolate 读取后传入)。 static Future fromBuffer( Uint8List bytes, List labels, { int modelId = -1, String modelName = '', }) async { try { final interpreter = Interpreter.fromBuffer( bytes, options: InterpreterOptions()..threads = 4, ); return TfliteDetector._fromModel( interpreter, labels, modelId, modelName); } catch (_) { return null; } } /// 输出布局 [1, 4+nc, anchors] 取自模型本身,类别数不与 labels 文件长度耦合。 factory TfliteDetector._fromModel(Interpreter interpreter, List labels, int modelId, String modelName) { final shape = interpreter.getOutputTensor(0).shape; final numClasses = shape.length >= 3 && shape[1] > 4 ? shape[1] - 4 : labels.length; final numAnchors = shape.length >= 3 && shape[2] > 0 ? shape[2] : 2100; final inputShape = interpreter.getInputTensor(0).shape; final inputSize = inputShape.length >= 4 ? inputShape[3] : defaultInputSize; final output = List.generate( 1, (_) => List.generate( numClasses + 4, (_) => List.filled(numAnchors, 0), ), ); return TfliteDetector._(interpreter, labels, numClasses, numAnchors, output, inputSize, modelId, modelName); } /// 原始数据接口(后台 isolate 用,不依赖 CameraImage)。 /// 输出坐标统一反算为原图归一化空间(与 MotionDetector 一致), /// 否则 CENTER_CROP 裁剪偏移会让检测框系统性偏移。 List detectRaw({ required List planes, required List strides, required int width, required int height, required bool isBgra, bool rgbaOrder = false, }) { preprocess( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder); // 传原始字节视图而非 Float32List:tflite_flutter 会对非 ByteBuffer/Uint8List // 输入调用 resizeInputTensor(1 维 [1486848]),使 node 0 TRANSPOSE prepare 失败 _interpreter.run(_input.buffer.asUint8List(), _output); final dets = postprocess(); // 反算与 preprocess 的 scale/dx/dy 公式一致(704 输入空间 → 原图归一化) final scale = inputSize / width < inputSize / height ? inputSize / width : inputSize / height; final dx = (inputSize - width * scale) / 2; final dy = (inputSize - height * scale) / 2; if (dx == 0 && dy == 0) return dets; return dets .map((r) => r.copyWith( left: (r.left * inputSize - dx) / (width * scale), right: (r.right * inputSize - dx) / (width * scale), top: (r.top * inputSize - dy) / (height * scale), bottom: (r.bottom * inputSize - dy) / (height * scale), )) .toList(); } /// 按像素格式分派:单平面 RGBA/BGRA(iOS bgra8888 / Android 实验) / yuv420 多平面。 void preprocess({ required List planes, required List strides, required int width, required int height, required bool isBgra, bool rgbaOrder = false, }) { if (isBgra) { _preprocessBgra(planes[0], strides[0], width, height, rgbaOrder); } else { _preprocessYuv(planes, strides, width, height); } } /// 单平面 8888(iOS bgra8888 = [b,g,r,a];Android 实验 RGBA_8888 = [r,g,b,a]): /// 每像素 4 字节,双线性采样,letterbox(等比缩到长边 704,短边黑边补 0)。 void _preprocessBgra( Uint8List src, int stride, int srcW, int srcH, bool rgbaOrder) { final plane = inputSize * inputSize; final scale = inputSize / srcW < inputSize / srcH ? inputSize / srcW : inputSize / srcH; final dx = (inputSize - srcW * scale) / 2; final dy = (inputSize - srcH * scale) / 2; // rgbaOrder=false(iOS BGRA): +0 B、+1 G、+2 R、+3 A; // rgbaOrder=true(Android RGBA): +0 R、+1 G、+2 B、+3 A final rOff = rgbaOrder ? 0 : 2; final bOff = rgbaOrder ? 2 : 0; for (var oy = 0; oy < inputSize; oy++) { final syf = (oy - dy) / scale; if (syf < 0 || syf >= srcH) { for (var ox = 0; ox < inputSize; ox++) { final p = oy * inputSize + ox; _input[p] = 0; _input[p + plane] = 0; _input[p + 2 * plane] = 0; } continue; } for (var ox = 0; ox < inputSize; ox++) { final p = oy * inputSize + ox; final sxf = (ox - dx) / scale; if (sxf < 0 || sxf >= srcW) { _input[p] = 0; _input[p + plane] = 0; _input[p + 2 * plane] = 0; continue; } final x0 = sxf.floor(), y0 = syf.floor(); final x1 = x0 < srcW - 1 ? x0 + 1 : x0; final y1 = y0 < srcH - 1 ? y0 + 1 : y0; final fx = sxf - x0, fy = syf - y0; final i00 = y0 * stride + x0 * 4; final i10 = y0 * stride + x1 * 4; final i01 = y1 * stride + x0 * 4; final i11 = y1 * stride + x1 * 4; final r00 = src[i00 + rOff].toDouble(); final g00 = src[i00 + 1].toDouble(); final b00 = src[i00 + bOff].toDouble(); final r10 = src[i10 + rOff].toDouble(); final g10 = src[i10 + 1].toDouble(); final b10 = src[i10 + bOff].toDouble(); final r01 = src[i01 + rOff].toDouble(); final g01 = src[i01 + 1].toDouble(); final b01 = src[i01 + bOff].toDouble(); final r11 = src[i11 + rOff].toDouble(); final g11 = src[i11 + 1].toDouble(); final b11 = src[i11 + bOff].toDouble(); _input[p] = _bl(r00, r10, r01, r11, fx, fy) / 255.0; _input[p + plane] = _bl(g00, g10, g01, g11, fx, fy) / 255.0; _input[p + 2 * plane] = _bl(b00, b10, b01, b11, fx, fy) / 255.0; } } } /// letterbox 缩放 + YUV → RGB 归一化 0~1(NCHW),双线性采样。 /// 兼容 NV12(双平面,UV 交错)与 I420(三平面)。 /// 首帧自适应:Y 值域(full/limited)与色序(U 先/V 先)因设备而异, /// 静态假设会在部分机型上产生偏色 → 检测退化。 void _preprocessYuv( List planes, List strides, int srcW, int srcH) { final plane = inputSize * inputSize; final y = planes[0]; final nv12 = planes.length == 2; final uv = nv12 ? planes[1] : null; final yStride = strides[0]; final uvStride = strides[1]; final vStride = nv12 ? uvStride : (strides.length > 2 ? strides[2] : strides[1]); // 色序修正后的 U/V 采样(nv12:偶位 U 奇位 V,NV21 相反;i420:平面 1/2 对调) double uAt(int x, int y) => nv12 ? uv![y * uvStride + (_yuvSwapChroma ? x * 2 + 1 : x * 2)] - 128.0 : planes[_yuvSwapChroma ? 2 : 1][y * uvStride + x] - 128.0; double vAt(int x, int y) => nv12 ? uv![y * uvStride + (_yuvSwapChroma ? x * 2 : x * 2 + 1)] - 128.0 : planes[_yuvSwapChroma ? 1 : 2][y * vStride + x] - 128.0; final scale = inputSize / srcW < inputSize / srcH ? inputSize / srcW : inputSize / srcH; final dx = (inputSize - srcW * scale) / 2; final dy = (inputSize - srcH * scale) / 2; for (var oy = 0; oy < inputSize; oy++) { final syf = (oy - dy) / scale; if (syf < 0 || syf >= srcH) { for (var ox = 0; ox < inputSize; ox++) { final p = oy * inputSize + ox; _input[p] = 0; _input[p + plane] = 0; _input[p + 2 * plane] = 0; } continue; } for (var ox = 0; ox < inputSize; ox++) { final p = oy * inputSize + ox; final sxf = (ox - dx) / scale; if (sxf < 0 || sxf >= srcW) { _input[p] = 0; _input[p + plane] = 0; _input[p + 2 * plane] = 0; continue; } final x0 = sxf.floor(), y0 = syf.floor(); final x1 = x0 < srcW - 1 ? x0 + 1 : x0; final y1 = y0 < srcH - 1 ? y0 + 1 : y0; final fx = sxf - x0, fy = syf - y0; // Y 双线性 final y00 = y[y0 * yStride + x0].toDouble(); final y10 = y[y0 * yStride + x1].toDouble(); final y01 = y[y1 * yStride + x0].toDouble(); final y11 = y[y1 * yStride + x1].toDouble(); final yy = _bl(y00, y10, y01, y11, fx, fy); // U/V 双线性(4:2:0 半分辨率,按像素坐标定位后除 2) final maxUx = srcW ~/ 2 - 1; final maxUy = srcH ~/ 2 - 1; final ux0 = (x0 ~/ 2).clamp(0, maxUx).toInt(); final uy0 = (y0 ~/ 2).clamp(0, maxUy).toInt(); final ux1 = (x1 ~/ 2).clamp(0, maxUx).toInt(); final uy1 = (y1 ~/ 2).clamp(0, maxUy).toInt(); final u00 = uAt(ux0, uy0); final u10 = uAt(ux1, uy0); final u01 = uAt(ux0, uy1); final u11 = uAt(ux1, uy1); final uu = _bl(u00, u10, u01, u11, fx, fy); final v00 = vAt(ux0, uy0); final v10 = vAt(ux1, uy0); final v01 = vAt(ux0, uy1); final v11 = vAt(ux1, uy1); final vv = _bl(v00, v10, v01, v11, fx, fy); // 值域展开:有限范围 VideoRange(Y 16~235,Cb/Cr 16~240)需线性拉伸; // 全值域相机直接使用原始值(与 iOS bgra 一致) final yr = _yuvFullRange ? yy : (yy - 16.0) * (255.0 / 219.0); final un = _yuvFullRange ? uu : uu * (255.0 / 224.0); final vn = _yuvFullRange ? vv : vv * (255.0 / 224.0); // NCHW:r/g/b 分平面存储 _input[p] = (yr + 1.402 * vn) / 255.0; _input[p + plane] = (yr - 0.344136 * un - 0.714136 * vn) / 255.0; _input[p + 2 * plane] = (yr + 1.772 * un) / 255.0; } } } /// 首帧自适应判定 YUV 模式,后续帧复用(相机重启后由 worker 复位重判)。 /// - 值域:有限范围黑电平恒为 16,低于 12 只可能是全值域。 /// - 色序:以模型本身为 oracle——同一帧按两种色序各推理一次, /// 检测数/最高分/总分更高者为真;两序均无检测时退回亮区色相计数启发 /// (户外最亮区域为天空应偏蓝,若按默认 U 先序解出偏红则为 V 先序)。 /// - 首帧可能曝光未收敛(过暗/全黑),此时 oracle 与启发式都不可信, /// 保持未判定状态等下一帧,避免在垃圾帧上锁死错误色序(真机零检测根因)。 bool _yuvFullRange = false; bool _yuvSwapChroma = false; bool _yuvModeKnown = false; bool _yuvRetried = false; int _yuvDecisionMs = 0; String _yuvDiag = ''; bool get yuvModeKnown => _yuvModeKnown; bool get yuvRetried => _yuvRetried; int get yuvDecisionMs => _yuvDecisionMs; String get yuvDiag => _yuvDiag; void decideYuvChroma({ required List planes, required List strides, required int width, required int height, bool force = false, }) { if (_yuvModeKnown && !force) return; final (yMin, yMax, yMean) = _yStats(planes[0], strides[0], width, height); _yuvFullRange = yMin < 12; if (yMean < 30 || yMax < 170) { // 曝光未稳定:保持未判定,下一帧重试;始终昏暗则维持默认(同旧版) if (!_yuvModeKnown) { _yuvDiag = '等稳定帧 mean=${yMean.toStringAsFixed(0)} max=$yMax'; } return; } _yuvModeKnown = true; _yuvDecisionMs = DateTime.now().millisecondsSinceEpoch; final a = _runWithSwap(planes, strides, width, height, false); final b = _runWithSwap(planes, strides, width, height, true); var swap = false; if (a.$1 != b.$1) { swap = b.$1 > a.$1; } else if (a.$2 != b.$2) { swap = b.$2 > a.$2; } else if (a.$3 != b.$3) { swap = b.$3 > a.$3; } else { swap = _brightRegionLeansRed(planes, strides, width, height, yMax); } _yuvSwapChroma = swap; _yuvDiag = 'full=$_yuvFullRange swap=$_yuvSwapChroma' ' cA=${a.$1} sA=${a.$2.toStringAsFixed(3)}' ' cB=${b.$1} sB=${b.$2.toStringAsFixed(3)}'; debugPrint('[yuv] $_yuvDiag mean=${yMean.toStringAsFixed(0)} max=$yMax'); } /// 判定后持续无检测的自愈:用实时帧重跑完整判定(仅一次)。 /// 首帧模糊/暗帧导致启发式猜错时,等画面稳定后 oracle 即可分胜负。 /// 返回是否执行了重判(随后应重跑 detectRaw 取新结果)。 bool retryDecision({ required List planes, required List strides, required int width, required int height, }) { if (_yuvRetried || !_yuvModeKnown) return false; final (_, yMax, yMean) = _yStats(planes[0], strides[0], width, height); if (yMean < 30 || yMax < 170) return false; // 帧仍不可用 _yuvRetried = true; decideYuvChroma( planes: planes, strides: strides, width: width, height: height, force: true); return true; } /// 相机(重新)启动后复位,首帧重新判定 void resetYuvMode() { _yuvModeKnown = false; _yuvRetried = false; _yuvDiag = ''; } /// 采样统计 Y 值域:(min, max, mean),步长 16px 约 3600 样本 (int, int, double) _yStats(Uint8List y, int yStride, int w, int h) { var yMin = 255, yMax = 0; var sum = 0, n = 0; for (var j = 0; j < h; j += 16) { final row = j * yStride; for (var i = 0; i < w; i += 16) { final v = y[row + i]; if (v < yMin) yMin = v; if (v > yMax) yMax = v; sum += v; n++; } } return (yMin, yMax, sum / n); } /// 按指定色序推理一次,返回 (检测数, 最高分, 总分) (int, double, double) _runWithSwap( List planes, List strides, int w, int h, bool swap) { _yuvSwapChroma = swap; preprocess(planes: planes, strides: strides, width: w, height: h, isBgra: false); _interpreter.run(_input.buffer.asUint8List(), _output); final dets = postprocess(); var maxScore = 0.0, sumScore = 0.0; for (final d in dets) { sumScore += d.score; if (d.score > maxScore) maxScore = d.score; } return (dets.length, maxScore, sumScore); } bool _brightRegionLeansRed( List planes, List strides, int w, int h, int yMax) { final y = planes[0]; final yStride = strides[0]; final uvStride = strides[1]; final nv12 = planes.length == 2; final uv = nv12 ? planes[1] : null; // 最亮带(maxY-40 以上),整体偏暗的场景也能拿到足量样本 final brightMin = yMax - 40; var blue = 0, red = 0; for (var j = 0; j < h; j += 8) { final yrow = j * yStride; for (var i = 0; i < w; i += 8) { if (y[yrow + i] < brightMin) continue; final cj = j ~/ 2, ci = i ~/ 2; if (nv12) { final c = cj * uvStride + ci * 2; if (c + 1 >= uv!.length) continue; if (uv[c] > 150) blue++; if (uv[c + 1] > 150) red++; } else { final c = cj * uvStride + ci; if (c >= planes[1].length || c >= planes[2].length) continue; if (planes[1][c] > 150) blue++; if (planes[2][c] > 150) red++; } } } // 亮区偏红多于偏蓝 → 当前 U/V 假设反了 return red > blue; } static double _bl(double a, double b, double c, double d, double fx, double fy) => (1 - fx) * (1 - fy) * a + fx * (1 - fy) * b + (1 - fx) * fy * c + fx * fy * d; List postprocess() { final out = _output[0]; final boxes = []; for (var a = 0; a < _numAnchors; a++) { final cx = out[0][a]; final cy = out[1][a]; final w = out[2][a]; final h = out[3][a]; var bestCls = 0; var bestScore = 0.0; for (var c = 0; c < _numClasses; c++) { final s = out[4 + c][a]; if (s > bestScore) { bestScore = s; bestCls = c; } } final label = bestCls < _labels.length ? _labels[bestCls] : 'unknown'; // 低分池保留,供运动检测提升显示 if (bestScore < minScore) continue; boxes.add(DetectionResult( label: label, score: bestScore, left: (cx - w / 2).clamp(0.0, 1.0), top: (cy - h / 2).clamp(0.0, 1.0), right: (cx + w / 2).clamp(0.0, 1.0), bottom: (cy + h / 2).clamp(0.0, 1.0), classId: bestCls, modelId: modelId, modelName: modelName, )); } final kept = nms(boxes, iouThreshold); return kept.take(maxDetections).toList(); } /// 诊断:按指定字节序推理一次,返回 (检测数, 最高分)。 /// 用于对比 BGRA/RGBA 两种顺序在同一帧上的检测差异(验证字节序与场景可达性)。 (int, double) diagnoseOrder({ required List planes, required List strides, required int width, required int height, required bool isBgra, required bool rgbaOrder, }) { preprocess( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder); _interpreter.run(_input.buffer.asUint8List(), _output); final dets = postprocess(); var maxScore = 0.0; for (final d in dets) { if (d.score > maxScore) maxScore = d.score; } return (dets.length, maxScore); } void dispose() => _interpreter.close(); }