import 'dart:async'; import 'dart:isolate'; import 'dart:typed_data'; import 'package:camera/camera.dart'; import 'package:flutter/foundation.dart' show debugPrint; import '../camera/motion_detector.dart'; import '../models/model_manager.dart'; import 'background_model.dart'; import 'detection_result.dart'; import 'nms.dart'; import 'tflite_detector.dart'; import 'visual_prior.dart'; /// 推理工作单元:模型加载与检测全部在后台 isolate 执行, /// 主 isolate 只投递帧数据、接收结果,UI 不被推理阻塞(iOS 真机卡顿根因)。 /// /// 多模型并行推理:传入 [models](各数据集下载模型)后,每帧逐模型推理, /// 结果跨模型全局 NMS 合并(2026-09-01 修订:异类别重叠也去重取高分,实测多模型对同一目标检异类别); /// 无下载模型时不启动推理(仅预览)。 class DetectorWorker { final Isolate _isolate; final ReceivePort _responses; final _controlPort = Completer(); final _ready = Completer(); SendPort? _port; /// 在途帧数(主 isolate 侧计数,用于丢帧) int _inFlight = 0; bool _dead = false; /// 结果回调:结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 / /// 处理耗时 ms / yuv 决策诊断串 void Function(List, List, List, int, int, int, int, String)? onResult; /// 单帧处理异常回调(不影响相机流) void Function(String)? onError; /// 最近一次创建失败的诊断原因(UI 展示用) static String? lastLoadError; /// worker 最近上报的执行步骤(诊断用) static String? lastLog; DetectorWorker._(this._isolate, this._responses) { _responses.listen(_onMessage, onDone: () { _dead = true; if (!_ready.isCompleted) { _ready.completeError(StateError('推理进程异常退出')); } onError?.call('推理进程异常退出'); }); } /// 加载模型并启动后台推理 isolate;无模型或加载失败返回 null(App 降级为仅预览)。 static Future create({List? models}) async { try { if (models == null || models.isEmpty) return null; final payload = >[ for (final m in models) [m.bytes, m.labels, m.datasetId, m.datasetName], ]; final responses = ReceivePort(); final isolate = await Isolate.spawn(_workerMain, responses.sendPort); final worker = DetectorWorker._(isolate, responses); final port = await worker._controlPort.future .timeout(const Duration(seconds: 10), onTimeout: () => throw TimeoutException('worker port timeout')); worker._port = port; port.send(['load', payload]); await worker._ready.future .timeout(const Duration(seconds: 20), onTimeout: () { throw TimeoutException('model load timeout'); }); return worker; } catch (e) { lastLoadError = e.toString(); debugPrint('[DetectorWorker] create failed: $e'); return null; } } /// 是否忙(上一帧尚未返回):忙则丢帧,避免在途积压 bool get busy => _inFlight > 0; void analyze(CameraImage image, int rotationDegrees, {bool rgbaOrder = false}) { // 单平面 8888 判定:仅明确的 yuv420/nv21 走多平面 YUV 路径; // bgra8888 与 unknown(插件未识别 RGBA_8888 输出时)都按 4 字节像素处理 final group = image.format.group; analyzeRaw( planes: image.planes.map((p) => p.bytes).toList(), strides: image.planes.map((p) => p.bytesPerRow).toList(), width: image.width, height: image.height, isBgra: group != ImageFormatGroup.yuv420 && group != ImageFormatGroup.nv21, rgbaOrder: rgbaOrder, rotationDegrees: rotationDegrees, ); } /// 原始字节帧投递(截屏注入用:toImage 的 RGBA 字节直接进检测,不经 CameraImage) void analyzeRaw({ required List planes, required List strides, required int width, required int height, required bool isBgra, required bool rgbaOrder, required int rotationDegrees, }) { final port = _port; if (port == null || _dead) return; _inFlight++; port.send([ 'frame', [planes, strides, width, height, isBgra, rotationDegrees, rgbaOrder], ]); } void _onMessage(dynamic msg) { final list = msg as List; switch (list[0] as String) { case 'port': _controlPort.complete(list[1] as SendPort); break; case 'ready': _ready.complete(); break; case 'load-error': _ready.completeError(StateError( list.length > 1 ? list[1] as String : 'model load failed')); break; case 'result': _inFlight--; final dets = (list[4] as List).map((d) { final v = d as List; return DetectionResult( label: v[0] as String, score: v[1] as double, left: v[2] as double, top: v[3] as double, right: v[4] as double, bottom: v[5] as double, modelId: v.length > 6 ? (v[6] as num).toInt() : -1, modelName: v.length > 7 ? v[7] as String : '', classId: v.length > 8 ? (v[8] as num).toInt() : -1, ); }).toList(); final motion = (list[5] as List) .map((m) => m as List) .map((v) => MotionRegion( v[0] as double, v[1] as double, v[2] as double, v[3] as double)) .toList(); final novelty = (list[6] as List) .map((m) => m as List) .map((v) => MotionRegion( v[0] as double, v[1] as double, v[2] as double, v[3] as double)) .toList(); onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int, list[3] as int, list[7] as int, list[8] as String); break; case 'log': lastLog = list[1] as String; debugPrint('[DetectorWorker] $lastLog'); break; case 'error': _inFlight--; onError?.call(list[1] as String); break; } } /// 相机切换/场景变化后重置运动与背景参考 void reset() { final port = _port; if (port == null || _dead) return; port.send(['reset']); } /// 调整置信度阈值(设置页滑块,worker 内实时生效) void setMinScore(double v) { final port = _port; if (port == null || _dead) return; port.send(['set-min-score', v]); } void dispose() { _dead = true; _isolate.kill(priority: Isolate.immediate); _responses.close(); } } /// 后台 isolate 入口:串行处理 load / frame / reset 命令。 /// 所有回发必须走 [mainPort](主 isolate 的端口);control 是 worker 自己的 /// 收件箱,往 control.sendPort 发消息等于发给自己,主 isolate 永远收不到。 Future _workerMain(SendPort mainPort) async { final control = ReceivePort(); mainPort.send(['port', control.sendPort]); mainPort.send(['log', 'worker-start']); List detectors = const []; MotionDetector? motion; BackgroundModel? background; var lastDualMs = 0; // 双字节序推理诊断节流 Uint8List? prevY; // 上一帧 Y/RGBA 平面(帧间 diff 诊断) await for (final msg in control) { try { final list = msg as List; switch (list[0] as String) { case 'load': mainPort.send(['log', 'load-received']); try { // 多模型:逐模型加载,单个失败不阻塞其余;全部失败才报错 final loaded = []; final failures = []; for (final entry in list[1] as List) { final e = entry as List; final name = e.length > 3 ? e[3] as String : ''; final d = await TfliteDetector.fromBuffer( e[0] as Uint8List, (e[1] as List).cast(), modelId: (e[2] as num).toInt(), modelName: name, ); if (d == null) { failures.add(name.isEmpty ? 'unknown' : name); } else { loaded.add(d); } } if (loaded.isEmpty) { mainPort.send([ 'load-error', '模型加载失败:${failures.join(',')} ' '(fromBuffer 返回 null)' ]); } else { detectors = loaded; mainPort.send(['log', 'loaded=${loaded.map((d) => d.modelName).join(',')} ' 'failed=${failures.isEmpty ? '-' : failures.join(',')}']); motion = MotionDetector(); background = BackgroundModel(); mainPort.send(['ready']); } } catch (e) { mainPort.send(['load-error', '$e']); } break; case 'frame': final m = motion; final b = background; if (detectors.isEmpty || m == null || b == null) break; final frame = list[1] as List; final planes = (frame[0] as List).cast(); final strides = (frame[1] as List).cast(); final width = frame[2] as int; final height = frame[3] as int; final isBgra = frame[4] as bool; final rotation = frame[5] as int; final rgbaOrder = frame.length > 6 && (frame[6] as bool); // 止血:非法帧(宽高/平面为空)直接丢弃并上报诊断, // 避免下游组件越界(RGBA patch 后插件偶发 w/h=0 帧) if (width <= 0 || height <= 0 || planes.isEmpty || planes[0].isEmpty) { mainPort.send([ 'error', 'bad frame w=$width h=$height planes=${planes.length} ' 'p0=${planes.isNotEmpty ? planes[0].length : 0} ' 'stride=${strides.isNotEmpty ? strides[0] : '-'} ' 'bgra=$isBgra' ]); break; } final sw = Stopwatch()..start(); // 帧内容统计(诊断):Y/RGBA 平面 min/max/mean + 与上帧的平均绝对差。 // 均匀灰帧 → min≈max≈mean;静止灰帧 → diff≈0;真实画面 → 分布宽且 diff>0 final yPlane = planes[0]; var yMin = 255, yMax = 0, ySum = 0, diff = 0, sampled = 0; final prev = prevY; for (var i = 0; i < yPlane.length; i += 8) { final v = yPlane[i]; if (v < yMin) yMin = v; if (v > yMax) yMax = v; ySum += v; if (prev != null && i < prev.length) { final d = v - prev[i]; diff += d < 0 ? -d : d; } sampled++; } prevY = yPlane; final yMean = ySum / sampled; final yDiff = prev == null ? -1.0 : diff / (sampled * 255.0); // UV 平面统计(诊断):色序/值域异常会导致解码偏色 var uv1 = '-'; if (planes.length > 1) { final u = planes[1]; var uMin = 255, uMax = 0, uSum = 0, n = 0; for (var i = 0; i < u.length; i += 8) { final v = u[i]; if (v < uMin) uMin = v; if (v > uMax) uMax = v; uSum += v; n++; } uv1 = 's=${strides[1]} min=$uMin max=$uMax mean=${(uSum / n).toStringAsFixed(0)}'; if (planes.length > 2) { final v2 = planes[2]; var vMin = 255, vMax = 0, vSum = 0, n2 = 0; for (var i = 0; i < v2.length; i += 8) { final v = v2[i]; if (v < vMin) vMin = v; if (v > vMax) vMax = v; vSum += v; n2++; } uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}'; } } // 多模型并行推理:每模型先首帧自适应判定 YUV 值域/色序,再逐模型推理; // 汇总后全局 NMS 合并(2026-09-01:异类别重叠也去重取高分,实测多模型对同一目标检异类别) var results = []; for (final d in detectors) { if (!isBgra && !d.yuvModeKnown) { d.decideYuvChroma( planes: planes, strides: strides, width: width, height: height, ); } var dets = d.detectRaw( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder, ); // 自愈:判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定 // (首帧模糊/暗帧导致启发式猜错时,画面稳定后 oracle 可分胜负) if (!isBgra && d.yuvModeKnown && !d.yuvRetried && dets.length <= 1 && DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs > 1500 && d.retryDecision( planes: planes, strides: strides, width: width, height: height, )) { dets = d.detectRaw( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, ); } results.addAll(dets); } results = mergeAcrossModels(results, TfliteDetector.iouThreshold); // 低分环颈雉鸡框过视觉先验(颜色/位置),减少户外误报 results = VisualPrior.filter( results, planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder, ); final motionRegions = m.detectMotionRaw( planes[0], strides[0], width, height); final noveltyRegions = b.updateRaw(planes[0], strides[0], width, height); sw.stop(); var dualDiag = ''; if (isBgra && DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) { lastDualMs = DateTime.now().millisecondsSinceEpoch; final a = detectors.first.diagnoseOrder( planes: planes, strides: strides, width: width, height: height, isBgra: true, rgbaOrder: false); final b = detectors.first.diagnoseOrder( planes: planes, strides: strides, width: width, height: height, isBgra: true, rgbaOrder: true); dualDiag = ' | dual BGRA:${a.$1}@${(a.$2 * 100).toStringAsFixed(1)}%' ' RGBA:${b.$1}@${(b.$2 * 100).toStringAsFixed(1)}%'; } mainPort.send([ 'result', rotation, width, height, results .map((r) => [ r.label, r.score, r.left, r.top, r.right, r.bottom, r.modelId, r.modelName, r.classId, ]) .toList(), motionRegions .map((mr) => [mr.left, mr.top, mr.right, mr.bottom]) .toList(), noveltyRegions .map((mr) => [mr.left, mr.top, mr.right, mr.bottom]) .toList(), sw.elapsedMilliseconds, 'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} ' 'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} ' 'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} ' 'uv1:[$uv1] | ${detectors.first.yuvDiag}$dualDiag', ]); break; case 'reset': motion?.reset(); background?.reset(); for (final d in detectors) { d.resetYuvMode(); } break; case 'set-min-score': for (final d in detectors) { d.minScore = (list[1] as num).toDouble(); } mainPort.send(['log', 'min-score=${detectors.isEmpty ? '-' : detectors.first.minScore}']); } } catch (e, st) { mainPort.send([ 'error', '$e\n${st.toString().split('\n').take(3).join('\n')}' ]); } } } /// 多模型结果合并:全局 NMS(不区分类别)。 /// 实测多个模型会对同一目标检出不同类别(误检/歧义),若异类别互不压制 /// 会出现重叠框;2026-09-01 用户实测定案:所有模型的框统一按 IoU 去重, /// 重叠时取高分(远处真实的多目标互不重叠,正常保留)。 List mergeAcrossModels( List all, double iouThreshold) { if (all.length <= 1) return all; return nms(all, iouThreshold); }