Files
observer/flutter_app/lib/detection/tflite_detector.dart
T
admin a0b115d954 训练体系整合与标注单阶段化
- 标注:AI 预标注直写 labels_json(去候选确认两阶段);重叠去重(minIoU);全量标注按钮
- 训练:脚本迁移入 server/training/(Go 化 prepare_yolo/analyze_rfdetr,保留 train_server.py);tflite 产物自检并入训练流程(check_tflite)
- 数据目录/权重不进 git;.gitignore 迁移至仓库根
2026-08-26 18:22:56 +08:00

520 lines
19 KiB
Dart
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import 'dart:typed_data';
import 'package:flutter/foundation.dart' show debugPrint;
import 'package:tflite_flutter/tflite_flutter.dart';
import 'detection_result.dart';
import 'nms.dart';
/// YOLOv8n 端侧推理实现(对应 Kotlin TFLiteDetector)。
/// 模型输出布局(ultralytics litert 导出):[1, 4 + nc, anchors]
/// cx/cy/w/h 已归一化,类别得分已过 sigmoid;按 out[dim][anchor] 索引。
/// 输入为 NCHW [1, 3, 704, 704]litert 导出保留 torch 布局)。
class TfliteDetector {
// 输入尺寸取自模型本身(ultralytics litert 导出 NCHW [1,3,H,W],各数据集
// 训练 imgsz 可不同),默认 704 兜底
static const int defaultInputSize = 704;
// 野鸡数据置信度普遍偏低(0.1~0.2 量级),保留低分池供运动检测提升;
// 可运行时调整(设置页滑块),默认 0.10
double minScore = 0.10;
static const double iouThreshold = 0.45;
static const int maxDetections = 20;
static const String modelAsset = 'assets/model.tflite';
static const String labelsAsset = 'assets/labels.txt';
final Interpreter _interpreter;
final List<String> _labels;
final int _numClasses;
final int _numAnchors;
final int inputSize;
/// 模型身份(多模型并行推理区分来源;内置资产模型为 -1/空)
final int modelId;
final String modelName;
late final Float32List _input;
/// 输出按模型形状 [1, 4+nc, anchors] 的嵌套 List 组织,
/// run() 要求输出对象形状与模型完全一致(扁平 List 会被拒)。
final List<List<List<double>>> _output;
TfliteDetector._(this._interpreter, this._labels, this._numClasses,
this._numAnchors, this._output, this.inputSize, this.modelId,
this.modelName) {
_input = Float32List(1 * inputSize * inputSize * 3);
}
/// 模型缺失或加载失败返回 null(App 降级为仅预览)。
/// 在后台 isolate 内调用(模型字节由主 isolate 读取后传入)。
static Future<TfliteDetector?> fromBuffer(
Uint8List bytes,
List<String> labels, {
int modelId = -1,
String modelName = '',
}) async {
try {
final interpreter = Interpreter.fromBuffer(
bytes,
options: InterpreterOptions()..threads = 4,
);
return TfliteDetector._fromModel(
interpreter, labels, modelId, modelName);
} catch (_) {
return null;
}
}
/// 输出布局 [1, 4+nc, anchors] 取自模型本身,类别数不与 labels 文件长度耦合。
factory TfliteDetector._fromModel(Interpreter interpreter,
List<String> labels, int modelId, String modelName) {
final shape = interpreter.getOutputTensor(0).shape;
final numClasses =
shape.length >= 3 && shape[1] > 4 ? shape[1] - 4 : labels.length;
final numAnchors = shape.length >= 3 && shape[2] > 0 ? shape[2] : 2100;
final inputShape = interpreter.getInputTensor(0).shape;
final inputSize = inputShape.length >= 4
? inputShape[3]
: defaultInputSize;
final output = List.generate(
1,
(_) => List.generate(
numClasses + 4,
(_) => List<double>.filled(numAnchors, 0),
),
);
return TfliteDetector._(interpreter, labels, numClasses, numAnchors,
output, inputSize, modelId, modelName);
}
/// 原始数据接口(后台 isolate 用,不依赖 CameraImage)。
/// 输出坐标统一反算为原图归一化空间(与 MotionDetector 一致),
/// 否则 CENTER_CROP 裁剪偏移会让检测框系统性偏移。
List<DetectionResult> detectRaw({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
bool rgbaOrder = false,
}) {
preprocess(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder);
// 传原始字节视图而非 Float32Listtflite_flutter 会对非 ByteBuffer/Uint8List
// 输入调用 resizeInputTensor1 维 [1486848]),使 node 0 TRANSPOSE prepare 失败
_interpreter.run(_input.buffer.asUint8List(), _output);
final dets = postprocess();
// 反算与 preprocess 的 scale/dx/dy 公式一致(704 输入空间 → 原图归一化)
final scale = inputSize / width < inputSize / height
? inputSize / width
: inputSize / height;
final dx = (inputSize - width * scale) / 2;
final dy = (inputSize - height * scale) / 2;
if (dx == 0 && dy == 0) return dets;
return dets
.map((r) => r.copyWith(
left: (r.left * inputSize - dx) / (width * scale),
right: (r.right * inputSize - dx) / (width * scale),
top: (r.top * inputSize - dy) / (height * scale),
bottom: (r.bottom * inputSize - dy) / (height * scale),
))
.toList();
}
/// 按像素格式分派:单平面 RGBA/BGRAiOS bgra8888 / Android 实验) / yuv420 多平面。
void preprocess({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
bool rgbaOrder = false,
}) {
if (isBgra) {
_preprocessBgra(planes[0], strides[0], width, height, rgbaOrder);
} else {
_preprocessYuv(planes, strides, width, height);
}
}
/// 单平面 8888iOS bgra8888 = [b,g,r,a]Android 实验 RGBA_8888 = [r,g,b,a]):
/// 每像素 4 字节,双线性采样,letterbox(等比缩到长边 704,短边黑边补 0)。
void _preprocessBgra(
Uint8List src, int stride, int srcW, int srcH, bool rgbaOrder) {
final plane = inputSize * inputSize;
final scale = inputSize / srcW < inputSize / srcH
? inputSize / srcW
: inputSize / srcH;
final dx = (inputSize - srcW * scale) / 2;
final dy = (inputSize - srcH * scale) / 2;
// rgbaOrder=falseiOS BGRA: +0 B、+1 G、+2 R、+3 A
// rgbaOrder=trueAndroid RGBA: +0 R、+1 G、+2 B、+3 A
final rOff = rgbaOrder ? 0 : 2;
final bOff = rgbaOrder ? 2 : 0;
for (var oy = 0; oy < inputSize; oy++) {
final syf = (oy - dy) / scale;
if (syf < 0 || syf >= srcH) {
for (var ox = 0; ox < inputSize; ox++) {
final p = oy * inputSize + ox;
_input[p] = 0;
_input[p + plane] = 0;
_input[p + 2 * plane] = 0;
}
continue;
}
for (var ox = 0; ox < inputSize; ox++) {
final p = oy * inputSize + ox;
final sxf = (ox - dx) / scale;
if (sxf < 0 || sxf >= srcW) {
_input[p] = 0;
_input[p + plane] = 0;
_input[p + 2 * plane] = 0;
continue;
}
final x0 = sxf.floor(), y0 = syf.floor();
final x1 = x0 < srcW - 1 ? x0 + 1 : x0;
final y1 = y0 < srcH - 1 ? y0 + 1 : y0;
final fx = sxf - x0, fy = syf - y0;
final i00 = y0 * stride + x0 * 4;
final i10 = y0 * stride + x1 * 4;
final i01 = y1 * stride + x0 * 4;
final i11 = y1 * stride + x1 * 4;
final r00 = src[i00 + rOff].toDouble();
final g00 = src[i00 + 1].toDouble();
final b00 = src[i00 + bOff].toDouble();
final r10 = src[i10 + rOff].toDouble();
final g10 = src[i10 + 1].toDouble();
final b10 = src[i10 + bOff].toDouble();
final r01 = src[i01 + rOff].toDouble();
final g01 = src[i01 + 1].toDouble();
final b01 = src[i01 + bOff].toDouble();
final r11 = src[i11 + rOff].toDouble();
final g11 = src[i11 + 1].toDouble();
final b11 = src[i11 + bOff].toDouble();
_input[p] = _bl(r00, r10, r01, r11, fx, fy) / 255.0;
_input[p + plane] = _bl(g00, g10, g01, g11, fx, fy) / 255.0;
_input[p + 2 * plane] = _bl(b00, b10, b01, b11, fx, fy) / 255.0;
}
}
}
/// letterbox 缩放 + YUV → RGB 归一化 0~1NCHW),双线性采样。
/// 兼容 NV12(双平面,UV 交错)与 I420(三平面)。
/// 首帧自适应:Y 值域(full/limited)与色序(U 先/V 先)因设备而异,
/// 静态假设会在部分机型上产生偏色 → 检测退化。
void _preprocessYuv(
List<Uint8List> planes, List<int> strides, int srcW, int srcH) {
final plane = inputSize * inputSize;
final y = planes[0];
final nv12 = planes.length == 2;
final uv = nv12 ? planes[1] : null;
final yStride = strides[0];
final uvStride = strides[1];
final vStride =
nv12 ? uvStride : (strides.length > 2 ? strides[2] : strides[1]);
// 色序修正后的 U/V 采样(nv12:偶位 U 奇位 VNV21 相反;i420:平面 1/2 对调)
double uAt(int x, int y) => nv12
? uv![y * uvStride + (_yuvSwapChroma ? x * 2 + 1 : x * 2)] - 128.0
: planes[_yuvSwapChroma ? 2 : 1][y * uvStride + x] - 128.0;
double vAt(int x, int y) => nv12
? uv![y * uvStride + (_yuvSwapChroma ? x * 2 : x * 2 + 1)] - 128.0
: planes[_yuvSwapChroma ? 1 : 2][y * vStride + x] - 128.0;
final scale = inputSize / srcW < inputSize / srcH
? inputSize / srcW
: inputSize / srcH;
final dx = (inputSize - srcW * scale) / 2;
final dy = (inputSize - srcH * scale) / 2;
for (var oy = 0; oy < inputSize; oy++) {
final syf = (oy - dy) / scale;
if (syf < 0 || syf >= srcH) {
for (var ox = 0; ox < inputSize; ox++) {
final p = oy * inputSize + ox;
_input[p] = 0;
_input[p + plane] = 0;
_input[p + 2 * plane] = 0;
}
continue;
}
for (var ox = 0; ox < inputSize; ox++) {
final p = oy * inputSize + ox;
final sxf = (ox - dx) / scale;
if (sxf < 0 || sxf >= srcW) {
_input[p] = 0;
_input[p + plane] = 0;
_input[p + 2 * plane] = 0;
continue;
}
final x0 = sxf.floor(), y0 = syf.floor();
final x1 = x0 < srcW - 1 ? x0 + 1 : x0;
final y1 = y0 < srcH - 1 ? y0 + 1 : y0;
final fx = sxf - x0, fy = syf - y0;
// Y 双线性
final y00 = y[y0 * yStride + x0].toDouble();
final y10 = y[y0 * yStride + x1].toDouble();
final y01 = y[y1 * yStride + x0].toDouble();
final y11 = y[y1 * yStride + x1].toDouble();
final yy = _bl(y00, y10, y01, y11, fx, fy);
// U/V 双线性(4:2:0 半分辨率,按像素坐标定位后除 2)
final maxUx = srcW ~/ 2 - 1;
final maxUy = srcH ~/ 2 - 1;
final ux0 = (x0 ~/ 2).clamp(0, maxUx).toInt();
final uy0 = (y0 ~/ 2).clamp(0, maxUy).toInt();
final ux1 = (x1 ~/ 2).clamp(0, maxUx).toInt();
final uy1 = (y1 ~/ 2).clamp(0, maxUy).toInt();
final u00 = uAt(ux0, uy0);
final u10 = uAt(ux1, uy0);
final u01 = uAt(ux0, uy1);
final u11 = uAt(ux1, uy1);
final uu = _bl(u00, u10, u01, u11, fx, fy);
final v00 = vAt(ux0, uy0);
final v10 = vAt(ux1, uy0);
final v01 = vAt(ux0, uy1);
final v11 = vAt(ux1, uy1);
final vv = _bl(v00, v10, v01, v11, fx, fy);
// 值域展开:有限范围 VideoRangeY 16~235Cb/Cr 16~240)需线性拉伸;
// 全值域相机直接使用原始值(与 iOS bgra 一致)
final yr = _yuvFullRange ? yy : (yy - 16.0) * (255.0 / 219.0);
final un = _yuvFullRange ? uu : uu * (255.0 / 224.0);
final vn = _yuvFullRange ? vv : vv * (255.0 / 224.0);
// NCHWr/g/b 分平面存储
_input[p] = (yr + 1.402 * vn) / 255.0;
_input[p + plane] = (yr - 0.344136 * un - 0.714136 * vn) / 255.0;
_input[p + 2 * plane] = (yr + 1.772 * un) / 255.0;
}
}
}
/// 首帧自适应判定 YUV 模式,后续帧复用(相机重启后由 worker 复位重判)。
/// - 值域:有限范围黑电平恒为 16,低于 12 只可能是全值域。
/// - 色序:以模型本身为 oracle——同一帧按两种色序各推理一次,
/// 检测数/最高分/总分更高者为真;两序均无检测时退回亮区色相计数启发
/// (户外最亮区域为天空应偏蓝,若按默认 U 先序解出偏红则为 V 先序)。
/// - 首帧可能曝光未收敛(过暗/全黑),此时 oracle 与启发式都不可信,
/// 保持未判定状态等下一帧,避免在垃圾帧上锁死错误色序(真机零检测根因)。
bool _yuvFullRange = false;
bool _yuvSwapChroma = false;
bool _yuvModeKnown = false;
bool _yuvRetried = false;
int _yuvDecisionMs = 0;
String _yuvDiag = '';
bool get yuvModeKnown => _yuvModeKnown;
bool get yuvRetried => _yuvRetried;
int get yuvDecisionMs => _yuvDecisionMs;
String get yuvDiag => _yuvDiag;
void decideYuvChroma({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
bool force = false,
}) {
if (_yuvModeKnown && !force) return;
final (yMin, yMax, yMean) = _yStats(planes[0], strides[0], width, height);
_yuvFullRange = yMin < 12;
if (yMean < 30 || yMax < 170) {
// 曝光未稳定:保持未判定,下一帧重试;始终昏暗则维持默认(同旧版)
if (!_yuvModeKnown) {
_yuvDiag = '等稳定帧 mean=${yMean.toStringAsFixed(0)} max=$yMax';
}
return;
}
_yuvModeKnown = true;
_yuvDecisionMs = DateTime.now().millisecondsSinceEpoch;
final a = _runWithSwap(planes, strides, width, height, false);
final b = _runWithSwap(planes, strides, width, height, true);
var swap = false;
if (a.$1 != b.$1) {
swap = b.$1 > a.$1;
} else if (a.$2 != b.$2) {
swap = b.$2 > a.$2;
} else if (a.$3 != b.$3) {
swap = b.$3 > a.$3;
} else {
swap = _brightRegionLeansRed(planes, strides, width, height, yMax);
}
_yuvSwapChroma = swap;
_yuvDiag = 'full=$_yuvFullRange swap=$_yuvSwapChroma'
' cA=${a.$1} sA=${a.$2.toStringAsFixed(3)}'
' cB=${b.$1} sB=${b.$2.toStringAsFixed(3)}';
debugPrint('[yuv] $_yuvDiag mean=${yMean.toStringAsFixed(0)} max=$yMax');
}
/// 判定后持续无检测的自愈:用实时帧重跑完整判定(仅一次)。
/// 首帧模糊/暗帧导致启发式猜错时,等画面稳定后 oracle 即可分胜负。
/// 返回是否执行了重判(随后应重跑 detectRaw 取新结果)。
bool retryDecision({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
}) {
if (_yuvRetried || !_yuvModeKnown) return false;
final (_, yMax, yMean) = _yStats(planes[0], strides[0], width, height);
if (yMean < 30 || yMax < 170) return false; // 帧仍不可用
_yuvRetried = true;
decideYuvChroma(
planes: planes, strides: strides, width: width, height: height,
force: true);
return true;
}
/// 相机(重新)启动后复位,首帧重新判定
void resetYuvMode() {
_yuvModeKnown = false;
_yuvRetried = false;
_yuvDiag = '';
}
/// 采样统计 Y 值域:(min, max, mean),步长 16px 约 3600 样本
(int, int, double) _yStats(Uint8List y, int yStride, int w, int h) {
var yMin = 255, yMax = 0;
var sum = 0, n = 0;
for (var j = 0; j < h; j += 16) {
final row = j * yStride;
for (var i = 0; i < w; i += 16) {
final v = y[row + i];
if (v < yMin) yMin = v;
if (v > yMax) yMax = v;
sum += v;
n++;
}
}
return (yMin, yMax, sum / n);
}
/// 按指定色序推理一次,返回 (检测数, 最高分, 总分)
(int, double, double) _runWithSwap(
List<Uint8List> planes, List<int> strides, int w, int h, bool swap) {
_yuvSwapChroma = swap;
preprocess(planes: planes, strides: strides, width: w, height: h,
isBgra: false);
_interpreter.run(_input.buffer.asUint8List(), _output);
final dets = postprocess();
var maxScore = 0.0, sumScore = 0.0;
for (final d in dets) {
sumScore += d.score;
if (d.score > maxScore) maxScore = d.score;
}
return (dets.length, maxScore, sumScore);
}
bool _brightRegionLeansRed(
List<Uint8List> planes, List<int> strides, int w, int h, int yMax) {
final y = planes[0];
final yStride = strides[0];
final uvStride = strides[1];
final nv12 = planes.length == 2;
final uv = nv12 ? planes[1] : null;
// 最亮带(maxY-40 以上),整体偏暗的场景也能拿到足量样本
final brightMin = yMax - 40;
var blue = 0, red = 0;
for (var j = 0; j < h; j += 8) {
final yrow = j * yStride;
for (var i = 0; i < w; i += 8) {
if (y[yrow + i] < brightMin) continue;
final cj = j ~/ 2, ci = i ~/ 2;
if (nv12) {
final c = cj * uvStride + ci * 2;
if (c + 1 >= uv!.length) continue;
if (uv[c] > 150) blue++;
if (uv[c + 1] > 150) red++;
} else {
final c = cj * uvStride + ci;
if (c >= planes[1].length || c >= planes[2].length) continue;
if (planes[1][c] > 150) blue++;
if (planes[2][c] > 150) red++;
}
}
}
// 亮区偏红多于偏蓝 → 当前 U/V 假设反了
return red > blue;
}
static double _bl(double a, double b, double c, double d, double fx,
double fy) =>
(1 - fx) * (1 - fy) * a + fx * (1 - fy) * b +
(1 - fx) * fy * c + fx * fy * d;
List<DetectionResult> postprocess() {
final out = _output[0];
final boxes = <DetectionResult>[];
for (var a = 0; a < _numAnchors; a++) {
final cx = out[0][a];
final cy = out[1][a];
final w = out[2][a];
final h = out[3][a];
var bestCls = 0;
var bestScore = 0.0;
for (var c = 0; c < _numClasses; c++) {
final s = out[4 + c][a];
if (s > bestScore) {
bestScore = s;
bestCls = c;
}
}
final label =
bestCls < _labels.length ? _labels[bestCls] : 'unknown';
// 低分池保留,供运动检测提升显示
if (bestScore < minScore) continue;
boxes.add(DetectionResult(
label: label,
score: bestScore,
left: (cx - w / 2).clamp(0.0, 1.0),
top: (cy - h / 2).clamp(0.0, 1.0),
right: (cx + w / 2).clamp(0.0, 1.0),
bottom: (cy + h / 2).clamp(0.0, 1.0),
modelId: modelId,
modelName: modelName,
));
}
final kept = nms(boxes, iouThreshold);
return kept.take(maxDetections).toList();
}
/// 诊断:按指定字节序推理一次,返回 (检测数, 最高分)。
/// 用于对比 BGRA/RGBA 两种顺序在同一帧上的检测差异(验证字节序与场景可达性)。
(int, double) diagnoseOrder({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
required bool rgbaOrder,
}) {
preprocess(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder);
_interpreter.run(_input.buffer.asUint8List(), _output);
final dets = postprocess();
var maxScore = 0.0;
for (final d in dets) {
if (d.score > maxScore) maxScore = d.score;
}
return (dets.length, maxScore);
}
void dispose() => _interpreter.close();
}