498 lines
18 KiB
Dart
498 lines
18 KiB
Dart
import 'dart:async';
|
||
import 'dart:isolate';
|
||
import 'dart:typed_data';
|
||
|
||
import 'package:camera/camera.dart';
|
||
import 'package:flutter/foundation.dart' show debugPrint;
|
||
import 'package:flutter/services.dart' show rootBundle;
|
||
|
||
import '../camera/motion_detector.dart';
|
||
import '../models/model_manager.dart';
|
||
import 'background_model.dart';
|
||
import 'detection_result.dart';
|
||
import 'nms.dart';
|
||
import 'tflite_detector.dart';
|
||
import 'visual_prior.dart';
|
||
|
||
/// 推理工作单元：模型加载与检测全部在后台 isolate 执行，
|
||
/// 主 isolate 只投递帧数据、接收结果，UI 不被推理阻塞（iOS 真机卡顿根因）。
|
||
///
|
||
/// 多模型并行推理：传入 [models]（各数据集下载模型）后，每帧逐模型推理，
|
||
/// 结果按类别分组跨模型 NMS 合并（同标签重复框取高分，不同标签互不压制）；
|
||
/// 无下载模型时回退内置资产模型。
|
||
class DetectorWorker {
|
||
static const String modelAsset = 'assets/model.tflite';
|
||
static const String labelsAsset = 'assets/labels.txt';
|
||
|
||
/// 内置资产回退模型的标识
|
||
static const int builtinModelId = -1;
|
||
static const String builtinModelName = '内置';
|
||
|
||
final Isolate _isolate;
|
||
final ReceivePort _responses;
|
||
|
||
final _controlPort = Completer<SendPort>();
|
||
final _ready = Completer<void>();
|
||
|
||
SendPort? _port;
|
||
|
||
/// 在途帧数（主 isolate 侧计数，用于丢帧）
|
||
int _inFlight = 0;
|
||
bool _dead = false;
|
||
|
||
/// 结果回调：结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 /
|
||
/// 处理耗时 ms / yuv 决策诊断串
|
||
void Function(List<DetectionResult>, List<MotionRegion>, List<MotionRegion>,
|
||
int, int, int, int, String)? onResult;
|
||
|
||
/// 单帧处理异常回调（不影响相机流）
|
||
void Function(String)? onError;
|
||
|
||
/// 最近一次创建失败的诊断原因（UI 展示用）
|
||
static String? lastLoadError;
|
||
|
||
/// worker 最近上报的执行步骤（诊断用）
|
||
static String? lastLog;
|
||
|
||
DetectorWorker._(this._isolate, this._responses) {
|
||
_responses.listen(_onMessage, onDone: () {
|
||
_dead = true;
|
||
if (!_ready.isCompleted) {
|
||
_ready.completeError(StateError('推理进程异常退出'));
|
||
}
|
||
onError?.call('推理进程异常退出');
|
||
});
|
||
}
|
||
|
||
/// 加载模型并启动后台推理 isolate；加载失败返回 null（App 降级为仅预览）。
|
||
/// [models] 为空时回退内置资产模型（模型缺失同样返回 null）。
|
||
static Future<DetectorWorker?> create({List<ModelBundle>? models}) async {
|
||
try {
|
||
final payload = <List<Object?>>[];
|
||
if (models != null && models.isNotEmpty) {
|
||
for (final m in models) {
|
||
payload.add(
|
||
[m.bytes, m.labels, m.datasetId, m.datasetName]);
|
||
}
|
||
} else {
|
||
final data = await rootBundle.load(modelAsset);
|
||
final modelBytes =
|
||
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes);
|
||
final labels = (await rootBundle.loadString(labelsAsset))
|
||
.split('\n')
|
||
.where((l) => l.trim().isNotEmpty)
|
||
.toList();
|
||
payload.add([modelBytes, labels, builtinModelId, builtinModelName]);
|
||
}
|
||
|
||
final responses = ReceivePort();
|
||
final isolate = await Isolate.spawn(_workerMain, responses.sendPort);
|
||
final worker = DetectorWorker._(isolate, responses);
|
||
|
||
final port = await worker._controlPort.future
|
||
.timeout(const Duration(seconds: 10),
|
||
onTimeout: () => throw TimeoutException('worker port timeout'));
|
||
worker._port = port;
|
||
port.send(['load', payload]);
|
||
await worker._ready.future
|
||
.timeout(const Duration(seconds: 20), onTimeout: () {
|
||
throw TimeoutException('model load timeout');
|
||
});
|
||
return worker;
|
||
} catch (e) {
|
||
lastLoadError = e.toString();
|
||
debugPrint('[DetectorWorker] create failed: $e');
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/// 是否忙（上一帧尚未返回）：忙则丢帧，避免在途积压
|
||
bool get busy => _inFlight > 0;
|
||
|
||
void analyze(CameraImage image, int rotationDegrees, {bool rgbaOrder = false}) {
|
||
// 单平面 8888 判定：仅明确的 yuv420/nv21 走多平面 YUV 路径；
|
||
// bgra8888 与 unknown（插件未识别 RGBA_8888 输出时）都按 4 字节像素处理
|
||
final group = image.format.group;
|
||
analyzeRaw(
|
||
planes: image.planes.map((p) => p.bytes).toList(),
|
||
strides: image.planes.map((p) => p.bytesPerRow).toList(),
|
||
width: image.width,
|
||
height: image.height,
|
||
isBgra: group != ImageFormatGroup.yuv420 && group != ImageFormatGroup.nv21,
|
||
rgbaOrder: rgbaOrder,
|
||
rotationDegrees: rotationDegrees,
|
||
);
|
||
}
|
||
|
||
/// 原始字节帧投递（截屏注入用：toImage 的 RGBA 字节直接进检测，不经 CameraImage）
|
||
void analyzeRaw({
|
||
required List<Uint8List> planes,
|
||
required List<int> strides,
|
||
required int width,
|
||
required int height,
|
||
required bool isBgra,
|
||
required bool rgbaOrder,
|
||
required int rotationDegrees,
|
||
}) {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
_inFlight++;
|
||
port.send([
|
||
'frame',
|
||
[planes, strides, width, height, isBgra, rotationDegrees, rgbaOrder],
|
||
]);
|
||
}
|
||
|
||
void _onMessage(dynamic msg) {
|
||
final list = msg as List;
|
||
switch (list[0] as String) {
|
||
case 'port':
|
||
_controlPort.complete(list[1] as SendPort);
|
||
break;
|
||
case 'ready':
|
||
_ready.complete();
|
||
break;
|
||
case 'load-error':
|
||
_ready.completeError(StateError(
|
||
list.length > 1 ? list[1] as String : 'model load failed'));
|
||
break;
|
||
case 'result':
|
||
_inFlight--;
|
||
final dets = (list[4] as List).map((d) {
|
||
final v = d as List;
|
||
return DetectionResult(
|
||
label: v[0] as String,
|
||
score: v[1] as double,
|
||
left: v[2] as double,
|
||
top: v[3] as double,
|
||
right: v[4] as double,
|
||
bottom: v[5] as double,
|
||
modelId: v.length > 6 ? (v[6] as num).toInt() : -1,
|
||
modelName: v.length > 7 ? v[7] as String : '',
|
||
);
|
||
}).toList();
|
||
final motion = (list[5] as List)
|
||
.map((m) => m as List)
|
||
.map((v) => MotionRegion(
|
||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||
.toList();
|
||
final novelty = (list[6] as List)
|
||
.map((m) => m as List)
|
||
.map((v) => MotionRegion(
|
||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||
.toList();
|
||
onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int,
|
||
list[3] as int, list[7] as int, list[8] as String);
|
||
break;
|
||
case 'log':
|
||
lastLog = list[1] as String;
|
||
debugPrint('[DetectorWorker] $lastLog');
|
||
break;
|
||
case 'error':
|
||
_inFlight--;
|
||
onError?.call(list[1] as String);
|
||
break;
|
||
}
|
||
}
|
||
|
||
/// 相机切换/场景变化后重置运动与背景参考
|
||
void reset() {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
port.send(['reset']);
|
||
}
|
||
|
||
/// 调整置信度阈值（设置页滑块，worker 内实时生效）
|
||
void setMinScore(double v) {
|
||
final port = _port;
|
||
if (port == null || _dead) return;
|
||
port.send(['set-min-score', v]);
|
||
}
|
||
|
||
void dispose() {
|
||
_dead = true;
|
||
_isolate.kill(priority: Isolate.immediate);
|
||
_responses.close();
|
||
}
|
||
}
|
||
|
||
/// 后台 isolate 入口：串行处理 load / frame / reset 命令。
|
||
/// 所有回发必须走 [mainPort]（主 isolate 的端口）；control 是 worker 自己的
|
||
/// 收件箱，往 control.sendPort 发消息等于发给自己，主 isolate 永远收不到。
|
||
Future<void> _workerMain(SendPort mainPort) async {
|
||
final control = ReceivePort();
|
||
mainPort.send(['port', control.sendPort]);
|
||
mainPort.send(['log', 'worker-start']);
|
||
|
||
List<TfliteDetector> detectors = const [];
|
||
MotionDetector? motion;
|
||
BackgroundModel? background;
|
||
var lastDualMs = 0; // 双字节序推理诊断节流
|
||
Uint8List? prevY; // 上一帧 Y/RGBA 平面（帧间 diff 诊断）
|
||
await for (final msg in control) {
|
||
try {
|
||
final list = msg as List;
|
||
switch (list[0] as String) {
|
||
case 'load':
|
||
mainPort.send(['log', 'load-received']);
|
||
try {
|
||
// 多模型：逐模型加载，单个失败不阻塞其余；全部失败才报错
|
||
final loaded = <TfliteDetector>[];
|
||
final failures = <String>[];
|
||
for (final entry in list[1] as List) {
|
||
final e = entry as List;
|
||
final name = e.length > 3 ? e[3] as String : '';
|
||
final d = await TfliteDetector.fromBuffer(
|
||
e[0] as Uint8List,
|
||
(e[1] as List).cast<String>(),
|
||
modelId: (e[2] as num).toInt(),
|
||
modelName: name,
|
||
);
|
||
if (d == null) {
|
||
failures.add(name.isEmpty ? 'unknown' : name);
|
||
} else {
|
||
loaded.add(d);
|
||
}
|
||
}
|
||
if (loaded.isEmpty) {
|
||
mainPort.send([
|
||
'load-error',
|
||
'模型加载失败:${failures.join(',')} '
|
||
'(fromBuffer 返回 null)'
|
||
]);
|
||
} else {
|
||
detectors = loaded;
|
||
mainPort.send(['log',
|
||
'loaded=${loaded.map((d) => d.modelName).join(',')} '
|
||
'failed=${failures.isEmpty ? '-' : failures.join(',')}']);
|
||
motion = MotionDetector();
|
||
background = BackgroundModel();
|
||
mainPort.send(['ready']);
|
||
}
|
||
} catch (e) {
|
||
mainPort.send(['load-error', '$e']);
|
||
}
|
||
break;
|
||
case 'frame':
|
||
final m = motion;
|
||
final b = background;
|
||
if (detectors.isEmpty || m == null || b == null) break;
|
||
final frame = list[1] as List;
|
||
final planes = (frame[0] as List).cast<Uint8List>();
|
||
final strides = (frame[1] as List).cast<int>();
|
||
final width = frame[2] as int;
|
||
final height = frame[3] as int;
|
||
final isBgra = frame[4] as bool;
|
||
final rotation = frame[5] as int;
|
||
final rgbaOrder = frame.length > 6 && (frame[6] as bool);
|
||
|
||
// 止血：非法帧（宽高/平面为空）直接丢弃并上报诊断，
|
||
// 避免下游组件越界（RGBA patch 后插件偶发 w/h=0 帧）
|
||
if (width <= 0 || height <= 0 || planes.isEmpty || planes[0].isEmpty) {
|
||
mainPort.send([
|
||
'error',
|
||
'bad frame w=$width h=$height planes=${planes.length} '
|
||
'p0=${planes.isNotEmpty ? planes[0].length : 0} '
|
||
'stride=${strides.isNotEmpty ? strides[0] : '-'} '
|
||
'bgra=$isBgra'
|
||
]);
|
||
break;
|
||
}
|
||
|
||
final sw = Stopwatch()..start();
|
||
// 帧内容统计（诊断）：Y/RGBA 平面 min/max/mean + 与上帧的平均绝对差。
|
||
// 均匀灰帧 → min≈max≈mean；静止灰帧 → diff≈0；真实画面 → 分布宽且 diff>0
|
||
final yPlane = planes[0];
|
||
var yMin = 255, yMax = 0, ySum = 0, diff = 0, sampled = 0;
|
||
final prev = prevY;
|
||
for (var i = 0; i < yPlane.length; i += 8) {
|
||
final v = yPlane[i];
|
||
if (v < yMin) yMin = v;
|
||
if (v > yMax) yMax = v;
|
||
ySum += v;
|
||
if (prev != null && i < prev.length) {
|
||
final d = v - prev[i];
|
||
diff += d < 0 ? -d : d;
|
||
}
|
||
sampled++;
|
||
}
|
||
prevY = yPlane;
|
||
final yMean = ySum / sampled;
|
||
final yDiff =
|
||
prev == null ? -1.0 : diff / (sampled * 255.0);
|
||
// UV 平面统计（诊断）：色序/值域异常会导致解码偏色
|
||
var uv1 = '-';
|
||
if (planes.length > 1) {
|
||
final u = planes[1];
|
||
var uMin = 255, uMax = 0, uSum = 0, n = 0;
|
||
for (var i = 0; i < u.length; i += 8) {
|
||
final v = u[i];
|
||
if (v < uMin) uMin = v;
|
||
if (v > uMax) uMax = v;
|
||
uSum += v;
|
||
n++;
|
||
}
|
||
uv1 = 's=${strides[1]} min=$uMin max=$uMax mean=${(uSum / n).toStringAsFixed(0)}';
|
||
if (planes.length > 2) {
|
||
final v2 = planes[2];
|
||
var vMin = 255, vMax = 0, vSum = 0, n2 = 0;
|
||
for (var i = 0; i < v2.length; i += 8) {
|
||
final v = v2[i];
|
||
if (v < vMin) vMin = v;
|
||
if (v > vMax) vMax = v;
|
||
vSum += v;
|
||
n2++;
|
||
}
|
||
uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}';
|
||
}
|
||
}
|
||
// 多模型并行推理：每模型先首帧自适应判定 YUV 值域/色序，再逐模型推理；
|
||
// 汇总后按类别分组跨模型 NMS 合并（同标签重复框取高分，异标签互不压制）
|
||
var results = <DetectionResult>[];
|
||
for (final d in detectors) {
|
||
if (!isBgra && !d.yuvModeKnown) {
|
||
d.decideYuvChroma(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
);
|
||
}
|
||
var dets = d.detectRaw(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
rgbaOrder: rgbaOrder,
|
||
);
|
||
// 自愈：判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定
|
||
// （首帧模糊/暗帧导致启发式猜错时，画面稳定后 oracle 可分胜负）
|
||
if (!isBgra && d.yuvModeKnown && !d.yuvRetried &&
|
||
dets.length <= 1 &&
|
||
DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs >
|
||
1500 &&
|
||
d.retryDecision(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
)) {
|
||
dets = d.detectRaw(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
);
|
||
}
|
||
results.addAll(dets);
|
||
}
|
||
results = mergeAcrossModels(results, TfliteDetector.iouThreshold);
|
||
// 低分野鸡框过视觉先验（颜色/位置），减少户外误报
|
||
results = VisualPrior.filter(
|
||
results,
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: isBgra,
|
||
rgbaOrder: rgbaOrder,
|
||
);
|
||
final motionRegions = m.detectMotionRaw(
|
||
planes[0], strides[0], width, height);
|
||
final noveltyRegions =
|
||
b.updateRaw(planes[0], strides[0], width, height);
|
||
sw.stop();
|
||
|
||
var dualDiag = '';
|
||
if (isBgra &&
|
||
DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) {
|
||
lastDualMs = DateTime.now().millisecondsSinceEpoch;
|
||
final a = detectors.first.diagnoseOrder(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: true,
|
||
rgbaOrder: false);
|
||
final b = detectors.first.diagnoseOrder(
|
||
planes: planes,
|
||
strides: strides,
|
||
width: width,
|
||
height: height,
|
||
isBgra: true,
|
||
rgbaOrder: true);
|
||
dualDiag = ' | dual BGRA:${a.$1}@${(a.$2 * 100).toStringAsFixed(1)}%'
|
||
' RGBA:${b.$1}@${(b.$2 * 100).toStringAsFixed(1)}%';
|
||
}
|
||
|
||
mainPort.send([
|
||
'result',
|
||
rotation,
|
||
width,
|
||
height,
|
||
results
|
||
.map((r) => [
|
||
r.label,
|
||
r.score,
|
||
r.left,
|
||
r.top,
|
||
r.right,
|
||
r.bottom,
|
||
r.modelId,
|
||
r.modelName,
|
||
])
|
||
.toList(),
|
||
motionRegions
|
||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||
.toList(),
|
||
noveltyRegions
|
||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||
.toList(),
|
||
sw.elapsedMilliseconds,
|
||
'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} '
|
||
'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} '
|
||
'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} '
|
||
'uv1:[$uv1] | ${detectors.first.yuvDiag}$dualDiag',
|
||
]);
|
||
break;
|
||
case 'reset':
|
||
motion?.reset();
|
||
background?.reset();
|
||
for (final d in detectors) {
|
||
d.resetYuvMode();
|
||
}
|
||
break;
|
||
case 'set-min-score':
|
||
for (final d in detectors) {
|
||
d.minScore = (list[1] as num).toDouble();
|
||
}
|
||
mainPort.send(['log', 'min-score=${detectors.isEmpty ? '-' : detectors.first.minScore}']);
|
||
}
|
||
} catch (e, st) {
|
||
mainPort.send([
|
||
'error',
|
||
'$e\n${st.toString().split('\n').take(3).join('\n')}'
|
||
]);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 多模型结果合并：按类别分组，组内 NMS（不同模型检出同一目标时取高分）。
|
||
/// 各模型类别体系独立（如野鸡/疑似 vs 野兔/疑似），不同类别互不压制。
|
||
List<DetectionResult> mergeAcrossModels(
|
||
List<DetectionResult> all, double iouThreshold) {
|
||
if (all.length <= 1) return all;
|
||
final byLabel = <String, List<DetectionResult>>{};
|
||
for (final r in all) {
|
||
byLabel.putIfAbsent(r.label, () => []).add(r);
|
||
}
|
||
final merged = <DetectionResult>[];
|
||
for (final group in byLabel.values) {
|
||
merged.addAll(nms(group, iouThreshold));
|
||
}
|
||
merged.sort((a, b) => b.score.compareTo(a.score));
|
||
return merged;
|
||
}
|