483 lines
17 KiB
Dart
483 lines
17 KiB
Dart
import 'dart:async';
|
||
import 'dart:isolate';
|
||
import 'dart:typed_data';
|
||
|
||
import 'package:camera/camera.dart';
|
||
import 'package:flutter/foundation.dart' show debugPrint;
|
||
|
||
import '../camera/motion_detector.dart';
|
||
import '../models/model_manager.dart';
|
||
import 'background_model.dart';
|
||
import 'detection_result.dart';
|
||
import 'nms.dart';
|
||
import 'tflite_detector.dart';
|
||
import 'visual_prior.dart';
|
||
|
||
/// 推理工作单元：模型加载与检测全部在后台 isolate 执行，
|
||
/// 主 isolate 只投递帧数据、接收结果，UI 不被推理阻塞（iOS 真机卡顿根因）。
|
||
///
|
||
/// 多模型并行推理：传入 [models]（各数据集下载模型）后，每帧逐模型推理，
|
||
/// 结果跨模型全局 NMS 合并（2026-09-01 修订：异类别重叠也去重取高分，实测多模型对同一目标检异类别）；
|
||
/// 无下载模型时不启动推理（仅预览）。
|
||
class DetectorWorker {
|
||
final Isolate _isolate;
|
||
final ReceivePort _responses;
|
||
|
||
final _controlPort = Completer<SendPort>();
|
||
final _ready = Completer<void>();
|
||
|
||
SendPort? _port;
|
||
|
||
/// 在途帧数（主 isolate 侧计数，用于丢帧）
|
||
int _inFlight = 0;
|
||
bool _dead = false;
|
||
|
||
/// 结果回调：结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 /
|
||
/// 处理耗时 ms / yuv 决策诊断串
|
||
void Function(List<DetectionResult>, List<MotionRegion>, List<MotionRegion>,
|
||
int, int, int, int, String)? onResult;
|
||
|
||
/// 单帧处理异常回调（不影响相机流）
|
||
void Function(String)? onError;
|
||
|
||
/// 最近一次创建失败的诊断原因（UI 展示用）
|
||
static String? lastLoadError;
|
||
|
||
/// worker 最近上报的执行步骤（诊断用）
|
||
static String? lastLog;
|
||
|
||
DetectorWorker._(this._isolate, this._responses) {
|
||
_responses.listen(_onMessage, onDone: () {
|
||
_dead = true;
|
||
if (!_ready.isCompleted) {
|
||
_ready.completeError(StateError('推理进程异常退出'));
|
||
}
|
||
onError?.call('推理进程异常退出');
|
||
});
|
||
}
|
||
|
||
/// 加载模型并启动后台推理 isolate；无模型或加载失败返回 null（App 降级为仅预览）。
|
||
static Future<DetectorWorker?> create({List<ModelBundle>? models}) async {
|
||
try {
|
||
if (models == null || models.isEmpty) return null;
|
||
final payload = <List<Object?>>[
|
||
for (final m in models)
|
||
[
|
||
m.bytes,
|
||
m.labels,
|
||
m.datasetId,
|
||
m.datasetName,
|
||
m.variant,
|
||
],
|
||
];
|
||
|
||
final responses = ReceivePort();
|
||
final isolate = await Isolate.spawn(_workerMain, responses.sendPort);
|
||
final worker = DetectorWorker._(isolate, responses);
|
||
|
||
final port = await worker._controlPort.future
|
||
.timeout(const Duration(seconds: 10),
|
||
onTimeout: () => throw TimeoutException('worker port timeout'));
|
||
worker._port = port;
|
||
port.send(['load', payload]);
|
||
await worker._ready.future
|
||
.timeout(const Duration(seconds: 20), onTimeout: () {
|
||
throw TimeoutException('model load timeout');
|
||
});
|
||
return worker;
|
||
} catch (e) {
|
||
lastLoadError = e.toString();
|
||
debugPrint('[DetectorWorker] create failed: $e');
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/// 是否忙（上一帧尚未返回）：忙则丢帧，避免在途积压
|
||
bool get busy => _inFlight > 0;
|
||
|
||
void analyze(CameraImage image, int rotationDegrees, {bool rgbaOrder = false}) {
|
||
// 单平面 8888 判定：仅明确的 yuv420/nv21 走多平面 YUV 路径；
|
||
// bgra8888 与 unknown（插件未识别 RGBA_8888 输出时）都按 4 字节像素处理
|
||
final group = image.format.group;
|
||
analyzeRaw(
|
||
planes: image.planes.map((p) => p.bytes).toList(),
|
||
strides: image.planes.map((p) => p.bytesPerRow).toList(),
|
||
width: image.width,
|
||
height: image.height,
|
||
isBgra: group != ImageFormatGroup.yuv420 && group != ImageFormatGroup.nv21,
|
||
rgbaOrder: rgbaOrder,
|
||
rotationDegrees: rotationDegrees,
|
||
);
|
||
}
|
||
|
||
/// 原始字节帧投递（截屏注入用：toImage 的 RGBA 字节直接进检测，不经 CameraImage）
|
||
void analyzeRaw({
|
||
required List<Uint8List> planes,
|
||
required List<int> strides,
|
||
required int width,
|
||
required int height,
|
||
required bool isBgra,
|
||
required bool rgbaOrder,
|
||
required int rotationDegrees,
|
||
}) {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
_inFlight++;
|
||
port.send([
|
||
'frame',
|
||
[planes, strides, width, height, isBgra, rotationDegrees, rgbaOrder],
|
||
]);
|
||
}
|
||
|
||
void _onMessage(dynamic msg) {
|
||
final list = msg as List;
|
||
switch (list[0] as String) {
|
||
case 'port':
|
||
_controlPort.complete(list[1] as SendPort);
|
||
break;
|
||
case 'ready':
|
||
_ready.complete();
|
||
break;
|
||
case 'load-error':
|
||
_ready.completeError(StateError(
|
||
list.length > 1 ? list[1] as String : 'model load failed'));
|
||
break;
|
||
case 'result':
|
||
_inFlight--;
|
||
final dets = (list[4] as List).map((d) {
|
||
final v = d as List;
|
||
return DetectionResult(
|
||
label: v[0] as String,
|
||
score: v[1] as double,
|
||
left: v[2] as double,
|
||
top: v[3] as double,
|
||
right: v[4] as double,
|
||
bottom: v[5] as double,
|
||
modelId: v.length > 6 ? (v[6] as num).toInt() : -1,
|
||
modelName: v.length > 7 ? v[7] as String : '',
|
||
classId: v.length > 8 ? (v[8] as num).toInt() : -1,
|
||
);
|
||
}).toList();
|
||
final motion = (list[5] as List)
|
||
.map((m) => m as List)
|
||
.map((v) => MotionRegion(
|
||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||
.toList();
|
||
final novelty = (list[6] as List)
|
||
.map((m) => m as List)
|
||
.map((v) => MotionRegion(
|
||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||
.toList();
|
||
onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int,
|
||
list[3] as int, list[7] as int, list[8] as String);
|
||
break;
|
||
case 'log':
|
||
lastLog = list[1] as String;
|
||
debugPrint('[DetectorWorker] $lastLog');
|
||
break;
|
||
case 'error':
|
||
_inFlight--;
|
||
onError?.call(list[1] as String);
|
||
break;
|
||
}
|
||
}
|
||
|
||
/// 相机切换/场景变化后重置运动与背景参考
|
||
void reset() {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
port.send(['reset']);
|
||
}
|
||
|
||
/// 调整置信度阈值（设置页滑块，worker 内实时生效）
|
||
void setMinScore(double v) {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
port.send(['set-min-score', v]);
|
||
}
|
||
|
||
void dispose() {
|
||
_dead = true;
|
||
_isolate.kill(priority: Isolate.immediate);
|
||
_responses.close();
|
||
}
|
||
}
|
||
|
||
/// 后台 isolate 入口：串行处理 load / frame / reset 命令。
|
||
/// 所有回发必须走 [mainPort]（主 isolate 的端口）；control 是 worker 自己的
|
||
/// 收件箱，往 control.sendPort 发消息等于发给自己，主 isolate 永远收不到。
|
||
Future<void> _workerMain(SendPort mainPort) async {
|
||
final control = ReceivePort();
|
||
mainPort.send(['port', control.sendPort]);
|
||
mainPort.send(['log', 'worker-start']);
|
||
|
||
List<TfliteDetector> detectors = const [];
|
||
MotionDetector? motion;
|
||
BackgroundModel? background;
|
||
var lastDualMs = 0; // 双字节序推理诊断节流
|
||
Uint8List? prevY; // 上一帧 Y/RGBA 平面（帧间 diff 诊断）
|
||
await for (final msg in control) {
|
||
try {
|
||
final list = msg as List;
|
||
switch (list[0] as String) {
|
||
case 'load':
|
||
mainPort.send(['log', 'load-received']);
|
||
try {
|
||
// 多模型：逐模型加载，单个失败不阻塞其余；全部失败才报错
|
||
final loaded = <TfliteDetector>[];
|
||
final failures = <String>[];
|
||
for (final entry in list[1] as List) {
|
||
final e = entry as List;
|
||
final name = e.length > 3 ? e[3] as String : '';
|
||
// 模型名带档位标识（数据集名+档位，框来源可辨 s/n）
|
||
final variant = e.length > 4 ? e[4] as String : '';
|
||
final displayName =
|
||
variant.isEmpty ? name : '$name($variant)';
|
||
final d = await TfliteDetector.fromBuffer(
|
||
e[0] as Uint8List,
|
||
(e[1] as List).cast<String>(),
|
||
modelId: (e[2] as num).toInt(),
|
||
modelName: displayName,
|
||
);
|
||
if (d == null) {
|
||
failures.add(name.isEmpty ? 'unknown' : name);
|
||
} else {
|
||
loaded.add(d);
|
||
}
|
||
}
|
||
if (loaded.isEmpty) {
|
||
mainPort.send([
|
||
'load-error',
|
||
'模型加载失败:${failures.join(',')} '
|
||
'(fromBuffer 返回 null)'
|
||
]);
|
||
} else {
|
||
detectors = loaded;
|
||
mainPort.send(['log',
|
||
'loaded=${loaded.map((d) => d.modelName).join(',')} '
|
||
'failed=${failures.isEmpty ? '-' : failures.join(',')}']);
|
||
motion = MotionDetector();
|
||
background = BackgroundModel();
|
||
mainPort.send(['ready']);
|
||
}
|
||
} catch (e) {
|
||
mainPort.send(['load-error', '$e']);
|
||
}
|
||
break;
|
||
case 'frame':
|
||
final m = motion;
|
||
final b = background;
|
||
if (detectors.isEmpty || m == null || b == null) break;
|
||
final frame = list[1] as List;
|
||
final planes = (frame[0] as List).cast<Uint8List>();
|
||
final strides = (frame[1] as List).cast<int>();
|
||
final width = frame[2] as int;
|
||
final height = frame[3] as int;
|
||
final isBgra = frame[4] as bool;
|
||
final rotation = frame[5] as int;
|
||
final rgbaOrder = frame.length > 6 && (frame[6] as bool);
|
||
|
||
// 止血：非法帧（宽高/平面为空）直接丢弃并上报诊断，
|
||
// 避免下游组件越界（RGBA patch 后插件偶发 w/h=0 帧）
|
||
if (width <= 0 || height <= 0 || planes.isEmpty || planes[0].isEmpty) {
|
||
mainPort.send([
|
||
'error',
|
||
'bad frame w=$width h=$height planes=${planes.length} '
|
||
'p0=${planes.isNotEmpty ? planes[0].length : 0} '
|
||
'stride=${strides.isNotEmpty ? strides[0] : '-'} '
|
||
'bgra=$isBgra'
|
||
]);
|
||
break;
|
||
}
|
||
|
||
final sw = Stopwatch()..start();
|
||
// 帧内容统计（诊断）：Y/RGBA 平面 min/max/mean + 与上帧的平均绝对差。
|
||
// 均匀灰帧 → min≈max≈mean；静止灰帧 → diff≈0；真实画面 → 分布宽且 diff>0
|
||
final yPlane = planes[0];
|
||
var yMin = 255, yMax = 0, ySum = 0, diff = 0, sampled = 0;
|
||
final prev = prevY;
|
||
for (var i = 0; i < yPlane.length; i += 8) {
|
||
final v = yPlane[i];
|
||
if (v < yMin) yMin = v;
|
||
if (v > yMax) yMax = v;
|
||
ySum += v;
|
||
if (prev != null && i < prev.length) {
|
||
final d = v - prev[i];
|
||
diff += d < 0 ? -d : d;
|
||
}
|
||
sampled++;
|
||
}
|
||
prevY = yPlane;
|
||
final yMean = ySum / sampled;
|
||
final yDiff =
|
||
prev == null ? -1.0 : diff / (sampled * 255.0);
|
||
// UV 平面统计（诊断）：色序/值域异常会导致解码偏色
|
||
var uv1 = '-';
|
||
if (planes.length > 1) {
|
||
final u = planes[1];
|
||
var uMin = 255, uMax = 0, uSum = 0, n = 0;
|
||
for (var i = 0; i < u.length; i += 8) {
|
||
final v = u[i];
|
||
if (v < uMin) uMin = v;
|
||
if (v > uMax) uMax = v;
|
||
uSum += v;
|
||
n++;
|
||
}
|
||
uv1 = 's=${strides[1]} min=$uMin max=$uMax mean=${(uSum / n).toStringAsFixed(0)}';
|
||
if (planes.length > 2) {
|
||
final v2 = planes[2];
|
||
var vMin = 255, vMax = 0, vSum = 0, n2 = 0;
|
||
for (var i = 0; i < v2.length; i += 8) {
|
||
final v = v2[i];
|
||
if (v < vMin) vMin = v;
|
||
if (v > vMax) vMax = v;
|
||
vSum += v;
|
||
n2++;
|
||
}
|
||
uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}';
|
||
}
|
||
}
|
||
// 多模型并行推理：每模型先首帧自适应判定 YUV 值域/色序，再逐模型推理；
|
||
// 汇总后全局 NMS 合并（2026-09-01：异类别重叠也去重取高分，实测多模型对同一目标检异类别）
|
||
var results = <DetectionResult>[];
|
||
for (final d in detectors) {
|
||
if (!isBgra && !d.yuvModeKnown) {
|
||
d.decideYuvChroma(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
);
|
||
}
|
||
var dets = d.detectRaw(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
rgbaOrder: rgbaOrder,
|
||
);
|
||
// 自愈：判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定
|
||
// （首帧模糊/暗帧导致启发式猜错时，画面稳定后 oracle 可分胜负）
|
||
if (!isBgra && d.yuvModeKnown && !d.yuvRetried &&
|
||
dets.length <= 1 &&
|
||
DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs >
|
||
1500 &&
|
||
d.retryDecision(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
)) {
|
||
dets = d.detectRaw(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
);
|
||
}
|
||
results.addAll(dets);
|
||
}
|
||
results = mergeAcrossModels(results, TfliteDetector.iouThreshold);
|
||
// 低分目标框过视觉先验（颜色/位置），减少户外误报
|
||
results = VisualPrior.filter(
|
||
results,
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
rgbaOrder: rgbaOrder,
|
||
);
|
||
final motionRegions = m.detectMotionRaw(
|
||
planes[0], strides[0], width, height);
|
||
final noveltyRegions =
|
||
b.updateRaw(planes[0], strides[0], width, height);
|
||
sw.stop();
|
||
|
||
var dualDiag = '';
|
||
if (isBgra &&
|
||
DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) {
|
||
lastDualMs = DateTime.now().millisecondsSinceEpoch;
|
||
final a = detectors.first.diagnoseOrder(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: true,
|
||
rgbaOrder: false);
|
||
final b = detectors.first.diagnoseOrder(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: true,
|
||
rgbaOrder: true);
|
||
dualDiag = ' | dual BGRA:${a.$1}@${(a.$2 * 100).toStringAsFixed(1)}%'
|
||
' RGBA:${b.$1}@${(b.$2 * 100).toStringAsFixed(1)}%';
|
||
}
|
||
|
||
mainPort.send([
|
||
'result',
|
||
rotation,
|
||
width,
|
||
height,
|
||
results
|
||
.map((r) => [
|
||
r.label,
|
||
r.score,
|
||
r.left,
|
||
r.top,
|
||
r.right,
|
||
r.bottom,
|
||
r.modelId,
|
||
r.modelName,
|
||
r.classId,
|
||
])
|
||
.toList(),
|
||
motionRegions
|
||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||
.toList(),
|
||
noveltyRegions
|
||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||
.toList(),
|
||
sw.elapsedMilliseconds,
|
||
'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} '
|
||
'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} '
|
||
'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} '
|
||
'uv1:[$uv1] | ${detectors.first.yuvDiag}$dualDiag',
|
||
]);
|
||
break;
|
||
case 'reset':
|
||
motion?.reset();
|
||
background?.reset();
|
||
for (final d in detectors) {
|
||
d.resetYuvMode();
|
||
}
|
||
break;
|
||
case 'set-min-score':
|
||
for (final d in detectors) {
|
||
d.minScore = (list[1] as num).toDouble();
|
||
}
|
||
mainPort.send(['log', 'min-score=${detectors.isEmpty ? '-' : detectors.first.minScore}']);
|
||
}
|
||
} catch (e, st) {
|
||
mainPort.send([
|
||
'error',
|
||
'$e\n${st.toString().split('\n').take(3).join('\n')}'
|
||
]);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 多模型结果合并：全局 NMS（不区分类别）。
|
||
/// 实测多个模型会对同一目标检出不同类别（误检/歧义），若异类别互不压制
|
||
/// 会出现重叠框；2026-09-01 用户实测定案：所有模型的框统一按 IoU 去重，
|
||
/// 重叠时取高分（远处真实的多目标互不重叠，正常保留）。
|
||
List<DetectionResult> mergeAcrossModels(
|
||
List<DetectionResult> all, double iouThreshold) {
|
||
if (all.length <= 1) return all;
|
||
return nms(all, iouThreshold);
|
||
}
|