red-future/observer
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
admin
2026-08-26 18:15:54 +08:00
parent 54d343b739
commit a4568d8a55
79 changed files with 11264 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flutter_app/lib/detection/detection_result.dart
+8
View File
@@ -9,6 +9,10 @@ class DetectionResult {
/// 轨迹已确认（多帧稳定/高分/活动确认），false = 候选，渲染为虚线
final bool confirmed;
/// 产出该框的模型（数据集 id 与名称；内置资产模型为 -1/空）
final int modelId;
final String modelName;
const DetectionResult({
required this.label,
required this.score,
@@ -17,6 +21,8 @@ class DetectionResult {
required this.right,
required this.bottom,
this.confirmed = true,
this.modelId = -1,
this.modelName = '',
});
double get width => right - left;
@@ -40,6 +46,8 @@ class DetectionResult {
right: right ?? this.right,
bottom: bottom ?? this.bottom,
confirmed: confirmed ?? this.confirmed,
modelId: modelId,
modelName: modelName,
);
}
flutter_app/lib/detection/detector_worker.dart
+131 -51
View File
@@ -7,17 +7,27 @@ import 'package:flutter/foundation.dart' show debugPrint;
import 'package:flutter/services.dart' show rootBundle;
import '../camera/motion_detector.dart';
import '../models/model_manager.dart';
import 'background_model.dart';
import 'detection_result.dart';
import 'nms.dart';
import 'tflite_detector.dart';
import 'visual_prior.dart';
/// 推理工作单元：模型加载与检测全部在后台 isolate 执行，
/// 主 isolate 只投递帧数据、接收结果，UI 不被推理阻塞（iOS 真机卡顿根因）。
///
/// 多模型并行推理：传入 [models]（各数据集下载模型）后，每帧逐模型推理，
/// 结果按类别分组跨模型 NMS 合并（同标签重复框取高分，不同标签互不压制）；
/// 无下载模型时回退内置资产模型。
class DetectorWorker {
static const String modelAsset = 'assets/model.tflite';
static const String labelsAsset = 'assets/labels.txt';
/// 内置资产回退模型的标识
static const int builtinModelId = -1;
static const String builtinModelName = '内置';
final Isolate _isolate;
final ReceivePort _responses;
@@ -54,16 +64,26 @@ class DetectorWorker {
});
}
/// 读取模型资产并启动后台推理 isolate；加载失败返回 null（App 降级为仅预览）。
static Future<DetectorWorker?> create() async {
/// 加载模型并启动后台推理 isolate；加载失败返回 null（App 降级为仅预览）。
/// [models] 为空时回退内置资产模型（模型缺失同样返回 null）。
static Future<DetectorWorker?> create({List<ModelBundle>? models}) async {
try {
final data = await rootBundle.load(modelAsset);
final modelBytes =
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes);
final labels = (await rootBundle.loadString(labelsAsset))
.split('\n')
.where((l) => l.trim().isNotEmpty)
.toList();
final payload = <List<Object?>>[];
if (models != null && models.isNotEmpty) {
for (final m in models) {
payload.add(
[m.bytes, m.labels, m.datasetId, m.datasetName]);
}
} else {
final data = await rootBundle.load(modelAsset);
final modelBytes =
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes);
final labels = (await rootBundle.loadString(labelsAsset))
.split('\n')
.where((l) => l.trim().isNotEmpty)
.toList();
payload.add([modelBytes, labels, builtinModelId, builtinModelName]);
}
final responses = ReceivePort();
final isolate = await Isolate.spawn(_workerMain, responses.sendPort);
@@ -73,7 +93,7 @@ class DetectorWorker {
.timeout(const Duration(seconds: 10),
onTimeout: () => throw TimeoutException('worker port timeout'));
worker._port = port;
port.send(['load', modelBytes, labels]);
port.send(['load', payload]);
await worker._ready.future
.timeout(const Duration(seconds: 20), onTimeout: () {
throw TimeoutException('model load timeout');
@@ -147,6 +167,8 @@ class DetectorWorker {
top: v[3] as double,
right: v[4] as double,
bottom: v[5] as double,
modelId: v.length > 6 ? (v[6] as num).toInt() : -1,
modelName: v.length > 7 ? v[7] as String : '',
);
}).toList();
final motion = (list[5] as List)
@@ -202,7 +224,7 @@ Future<void> _workerMain(SendPort mainPort) async {
mainPort.send(['port', control.sendPort]);
mainPort.send(['log', 'worker-start']);
TfliteDetector? detector;
List<TfliteDetector> detectors = const [];
MotionDetector? motion;
BackgroundModel? background;
var lastDualMs = 0; // 双字节序推理诊断节流
@@ -214,12 +236,35 @@ Future<void> _workerMain(SendPort mainPort) async {
case 'load':
mainPort.send(['log', 'load-received']);
try {
detector = await TfliteDetector.fromBuffer(
list[1] as Uint8List, (list[2] as List).cast<String>());
if (detector == null) {
mainPort.send(['load-error', 'fromBuffer 返回 null']);
// 多模型：逐模型加载，单个失败不阻塞其余；全部失败才报错
final loaded = <TfliteDetector>[];
final failures = <String>[];
for (final entry in list[1] as List) {
final e = entry as List;
final name = e.length > 3 ? e[3] as String : '';
final d = await TfliteDetector.fromBuffer(
e[0] as Uint8List,
(e[1] as List).cast<String>(),
modelId: (e[2] as num).toInt(),
modelName: name,
);
if (d == null) {
failures.add(name.isEmpty ? 'unknown' : name);
} else {
loaded.add(d);
}
}
if (loaded.isEmpty) {
mainPort.send([
'load-error',
'模型加载失败:${failures.join(',')} '
'(fromBuffer 返回 null)'
]);
} else {
mainPort.send(['log', 'fromBuffer-ok']);
detectors = loaded;
mainPort.send(['log',
'loaded=${loaded.map((d) => d.modelName).join(',')} '
'failed=${failures.isEmpty ? '-' : failures.join(',')}']);
motion = MotionDetector();
background = BackgroundModel();
mainPort.send(['ready']);
@@ -229,10 +274,9 @@ Future<void> _workerMain(SendPort mainPort) async {
}
break;
case 'frame':
final d = detector;
final m = motion;
final b = background;
if (d == null || m == null || b == null) break;
if (detectors.isEmpty || m == null || b == null) break;
final frame = list[1] as List;
final planes = (frame[0] as List).cast<Uint8List>();
final strides = (frame[1] as List).cast<int>();
@@ -302,42 +346,49 @@ Future<void> _workerMain(SendPort mainPort) async {
uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}';
}
}
// 首帧（或相机重启后）自适应判定 YUV 值域与色序，再跑正式推理
if (!isBgra && !d.yuvModeKnown) {
d.decideYuvChroma(
planes: planes,
strides: strides,
width: width,
height: height,
);
}
var results = d.detectRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder,
);
// 自愈：判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定
// （首帧模糊/暗帧导致启发式猜错时，画面稳定后 oracle 可分胜负）
if (!isBgra && d.yuvModeKnown && !d.yuvRetried &&
results.length <= 1 &&
DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs > 1500 &&
d.retryDecision(
// 多模型并行推理：每模型先首帧自适应判定 YUV 值域/色序，再逐模型推理；
// 汇总后按类别分组跨模型 NMS 合并（同标签重复框取高分，异标签互不压制）
var results = <DetectionResult>[];
for (final d in detectors) {
if (!isBgra && !d.yuvModeKnown) {
d.decideYuvChroma(
planes: planes,
strides: strides,
width: width,
height: height,
)) {
results = d.detectRaw(
);
}
var dets = d.detectRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder,
);
// 自愈：判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定
// （首帧模糊/暗帧导致启发式猜错时，画面稳定后 oracle 可分胜负）
if (!isBgra && d.yuvModeKnown && !d.yuvRetried &&
dets.length <= 1 &&
DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs >
1500 &&
d.retryDecision(
planes: planes,
strides: strides,
width: width,
height: height,
)) {
dets = d.detectRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
);
}
results.addAll(dets);
}
results = mergeAcrossModels(results, TfliteDetector.iouThreshold);
// 低分野鸡框过视觉先验（颜色/位置），减少户外误报
results = VisualPrior.filter(
results,
@@ -358,14 +409,14 @@ Future<void> _workerMain(SendPort mainPort) async {
if (isBgra &&
DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) {
lastDualMs = DateTime.now().millisecondsSinceEpoch;
final a = d.diagnoseOrder(
final a = detectors.first.diagnoseOrder(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: true,
rgbaOrder: false);
final b = d.diagnoseOrder(
final b = detectors.first.diagnoseOrder(
planes: planes,
strides: strides,
width: width,
@@ -382,8 +433,16 @@ Future<void> _workerMain(SendPort mainPort) async {
width,
height,
results
.map((r) =>
[r.label, r.score, r.left, r.top, r.right, r.bottom])
.map((r) => [
r.label,
r.score,
r.left,
r.top,
r.right,
r.bottom,
r.modelId,
r.modelName,
])
.toList(),
motionRegions
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
@@ -395,17 +454,21 @@ Future<void> _workerMain(SendPort mainPort) async {
'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} '
'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} '
'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} '
'uv1:[$uv1] | ${d.yuvDiag}$dualDiag',
'uv1:[$uv1] | ${detectors.first.yuvDiag}$dualDiag',
]);
break;
case 'reset':
motion?.reset();
background?.reset();
detector?.resetYuvMode();
for (final d in detectors) {
d.resetYuvMode();
}
break;
case 'set-min-score':
detector?.minScore = (list[1] as num).toDouble();
mainPort.send(['log', 'min-score=${detector?.minScore}']);
for (final d in detectors) {
d.minScore = (list[1] as num).toDouble();
}
mainPort.send(['log', 'min-score=${detectors.isEmpty ? '-' : detectors.first.minScore}']);
}
} catch (e, st) {
mainPort.send([
@@ -415,3 +478,20 @@ Future<void> _workerMain(SendPort mainPort) async {
}
}
}
/// 多模型结果合并：按类别分组，组内 NMS（不同模型检出同一目标时取高分）。
/// 各模型类别体系独立（如野鸡/疑似 vs 野兔/疑似），不同类别互不压制。
List<DetectionResult> mergeAcrossModels(
List<DetectionResult> all, double iouThreshold) {
if (all.length <= 1) return all;
final byLabel = <String, List<DetectionResult>>{};
for (final r in all) {
byLabel.putIfAbsent(r.label, () => []).add(r);
}
final merged = <DetectionResult>[];
for (final group in byLabel.values) {
merged.addAll(nms(group, iouThreshold));
}
merged.sort((a, b) => b.score.compareTo(a.score));
return merged;
}
flutter_app/lib/detection/tflite_detector.dart
+30 -10
View File
@@ -11,7 +11,9 @@ import 'nms.dart';
/// cx/cy/w/h 已归一化，类别得分已过 sigmoid；按 out[dim][anchor] 索引。
/// 输入为 NCHW [1, 3, 704, 704]（litert 导出保留 torch 布局）。
class TfliteDetector {
static const int inputSize = 704;
// 输入尺寸取自模型本身（ultralytics litert 导出 NCHW [1,3,H,W]，各数据集
// 训练 imgsz 可不同），默认 704 兜底
static const int defaultInputSize = 704;
// 野鸡数据置信度普遍偏低（0.1~0.2 量级），保留低分池供运动检测提升；
// 可运行时调整（设置页滑块），默认 0.10
double minScore = 0.10;
@@ -24,39 +26,55 @@ class TfliteDetector {
final List<String> _labels;
final int _numClasses;
final int _numAnchors;
final int inputSize;
final Float32List _input =
Float32List(1 * inputSize * inputSize * 3);
/// 模型身份（多模型并行推理区分来源；内置资产模型为 -1/空）
final int modelId;
final String modelName;
late final Float32List _input;
/// 输出按模型形状 [1, 4+nc, anchors] 的嵌套 List 组织，
/// run() 要求输出对象形状与模型完全一致（扁平 List 会被拒）。
final List<List<List<double>>> _output;
TfliteDetector._(this._interpreter, this._labels, this._numClasses,
this._numAnchors, this._output);
this._numAnchors, this._output, this.inputSize, this.modelId,
this.modelName) {
_input = Float32List(1 * inputSize * inputSize * 3);
}
/// 模型缺失或加载失败返回 null（App 降级为仅预览）。
/// 在后台 isolate 内调用（模型字节由主 isolate 读取后传入）。
static Future<TfliteDetector?> fromBuffer(
Uint8List bytes, List<String> labels) async {
Uint8List bytes,
List<String> labels, {
int modelId = -1,
String modelName = '',
}) async {
try {
final interpreter = Interpreter.fromBuffer(
bytes,
options: InterpreterOptions()..threads = 4,
);
return TfliteDetector._fromModel(interpreter, labels);
return TfliteDetector._fromModel(
interpreter, labels, modelId, modelName);
} catch (_) {
return null;
}
}
/// 输出布局 [1, 4+nc, anchors] 取自模型本身，类别数不与 labels 文件长度耦合。
factory TfliteDetector._fromModel(
Interpreter interpreter, List<String> labels) {
factory TfliteDetector._fromModel(Interpreter interpreter,
List<String> labels, int modelId, String modelName) {
final shape = interpreter.getOutputTensor(0).shape;
final numClasses =
shape.length >= 3 && shape[1] > 4 ? shape[1] - 4 : labels.length;
final numAnchors = shape.length >= 3 && shape[2] > 0 ? shape[2] : 2100;
final inputShape = interpreter.getInputTensor(0).shape;
final inputSize = inputShape.length >= 4
? inputShape[3]
: defaultInputSize;
final output = List.generate(
1,
(_) => List.generate(
@@ -64,8 +82,8 @@ class TfliteDetector {
(_) => List<double>.filled(numAnchors, 0),
),
);
return TfliteDetector._(
interpreter, labels, numClasses, numAnchors, output);
return TfliteDetector._(interpreter, labels, numClasses, numAnchors,
output, inputSize, modelId, modelName);
}
/// 原始数据接口（后台 isolate 用，不依赖 CameraImage）。
@@ -463,6 +481,8 @@ class TfliteDetector {
top: (cy - h / 2).clamp(0.0, 1.0),
right: (cx + w / 2).clamp(0.0, 1.0),
bottom: (cy + h / 2).clamp(0.0, 1.0),
modelId: modelId,
modelName: modelName,
));
}
final kept = nms(boxes, iouThreshold);