import 'dart:async'; import 'dart:isolate'; import 'dart:typed_data'; import 'package:camera/camera.dart'; import 'package:flutter/foundation.dart' show debugPrint; import 'package:flutter/services.dart' show rootBundle; import '../camera/motion_detector.dart'; import 'background_model.dart'; import 'detection_result.dart'; import 'tflite_detector.dart'; import 'visual_prior.dart'; /// 推理工作单元:模型加载与检测全部在后台 isolate 执行, /// 主 isolate 只投递帧数据、接收结果,UI 不被推理阻塞(iOS 真机卡顿根因)。 class DetectorWorker { static const String modelAsset = 'assets/model.tflite'; static const String labelsAsset = 'assets/labels.txt'; final Isolate _isolate; final ReceivePort _responses; final _controlPort = Completer(); final _ready = Completer(); SendPort? _port; /// 在途帧数(主 isolate 侧计数,用于丢帧) int _inFlight = 0; bool _dead = false; /// 结果回调:结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 / /// 处理耗时 ms / yuv 决策诊断串 void Function(List, List, List, int, int, int, int, String)? onResult; /// 单帧处理异常回调(不影响相机流) void Function(String)? onError; /// 最近一次创建失败的诊断原因(UI 展示用) static String? lastLoadError; /// worker 最近上报的执行步骤(诊断用) static String? lastLog; DetectorWorker._(this._isolate, this._responses) { _responses.listen(_onMessage, onDone: () { _dead = true; if (!_ready.isCompleted) { _ready.completeError(StateError('推理进程异常退出')); } onError?.call('推理进程异常退出'); }); } /// 读取模型资产并启动后台推理 isolate;加载失败返回 null(App 降级为仅预览)。 static Future create() async { try { final data = await rootBundle.load(modelAsset); final modelBytes = data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes); final labels = (await rootBundle.loadString(labelsAsset)) .split('\n') .where((l) => l.trim().isNotEmpty) .toList(); final responses = ReceivePort(); final isolate = await Isolate.spawn(_workerMain, responses.sendPort); final worker = DetectorWorker._(isolate, responses); final port = await worker._controlPort.future .timeout(const Duration(seconds: 10), onTimeout: () => throw TimeoutException('worker port timeout')); worker._port = port; port.send(['load', modelBytes, labels]); await worker._ready.future .timeout(const Duration(seconds: 20), onTimeout: () { throw TimeoutException('model load timeout'); }); return worker; } catch (e) { lastLoadError = e.toString(); debugPrint('[DetectorWorker] create failed: $e'); return null; } } /// 是否忙(上一帧尚未返回):忙则丢帧,避免在途积压 bool get busy => _inFlight > 0; void analyze(CameraImage image, int rotationDegrees, {bool rgbaOrder = false}) { // 单平面 8888 判定:仅明确的 yuv420/nv21 走多平面 YUV 路径; // bgra8888 与 unknown(插件未识别 RGBA_8888 输出时)都按 4 字节像素处理 final group = image.format.group; analyzeRaw( planes: image.planes.map((p) => p.bytes).toList(), strides: image.planes.map((p) => p.bytesPerRow).toList(), width: image.width, height: image.height, isBgra: group != ImageFormatGroup.yuv420 && group != ImageFormatGroup.nv21, rgbaOrder: rgbaOrder, rotationDegrees: rotationDegrees, ); } /// 原始字节帧投递(截屏注入用:toImage 的 RGBA 字节直接进检测,不经 CameraImage) void analyzeRaw({ required List planes, required List strides, required int width, required int height, required bool isBgra, required bool rgbaOrder, required int rotationDegrees, }) { final port = _port; if (port == null || _dead) return; _inFlight++; port.send([ 'frame', [planes, strides, width, height, isBgra, rotationDegrees, rgbaOrder], ]); } void _onMessage(dynamic msg) { final list = msg as List; switch (list[0] as String) { case 'port': _controlPort.complete(list[1] as SendPort); break; case 'ready': _ready.complete(); break; case 'load-error': _ready.completeError(StateError( list.length > 1 ? list[1] as String : 'model load failed')); break; case 'result': _inFlight--; final dets = (list[4] as List).map((d) { final v = d as List; return DetectionResult( label: v[0] as String, score: v[1] as double, left: v[2] as double, top: v[3] as double, right: v[4] as double, bottom: v[5] as double, ); }).toList(); final motion = (list[5] as List) .map((m) => m as List) .map((v) => MotionRegion( v[0] as double, v[1] as double, v[2] as double, v[3] as double)) .toList(); final novelty = (list[6] as List) .map((m) => m as List) .map((v) => MotionRegion( v[0] as double, v[1] as double, v[2] as double, v[3] as double)) .toList(); onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int, list[3] as int, list[7] as int, list[8] as String); break; case 'log': lastLog = list[1] as String; debugPrint('[DetectorWorker] $lastLog'); break; case 'error': _inFlight--; onError?.call(list[1] as String); break; } } /// 相机切换/场景变化后重置运动与背景参考 void reset() { final port = _port; if (port == null || _dead) return; port.send(['reset']); } /// 调整置信度阈值(设置页滑块,worker 内实时生效) void setMinScore(double v) { final port = _port; if (port == null || _dead) return; port.send(['set-min-score', v]); } void dispose() { _dead = true; _isolate.kill(priority: Isolate.immediate); _responses.close(); } } /// 后台 isolate 入口:串行处理 load / frame / reset 命令。 /// 所有回发必须走 [mainPort](主 isolate 的端口);control 是 worker 自己的 /// 收件箱,往 control.sendPort 发消息等于发给自己,主 isolate 永远收不到。 Future _workerMain(SendPort mainPort) async { final control = ReceivePort(); mainPort.send(['port', control.sendPort]); mainPort.send(['log', 'worker-start']); TfliteDetector? detector; MotionDetector? motion; BackgroundModel? background; var lastDualMs = 0; // 双字节序推理诊断节流 Uint8List? prevY; // 上一帧 Y/RGBA 平面(帧间 diff 诊断) await for (final msg in control) { try { final list = msg as List; switch (list[0] as String) { case 'load': mainPort.send(['log', 'load-received']); try { detector = await TfliteDetector.fromBuffer( list[1] as Uint8List, (list[2] as List).cast()); if (detector == null) { mainPort.send(['load-error', 'fromBuffer 返回 null']); } else { mainPort.send(['log', 'fromBuffer-ok']); motion = MotionDetector(); background = BackgroundModel(); mainPort.send(['ready']); } } catch (e) { mainPort.send(['load-error', '$e']); } break; case 'frame': final d = detector; final m = motion; final b = background; if (d == null || m == null || b == null) break; final frame = list[1] as List; final planes = (frame[0] as List).cast(); final strides = (frame[1] as List).cast(); final width = frame[2] as int; final height = frame[3] as int; final isBgra = frame[4] as bool; final rotation = frame[5] as int; final rgbaOrder = frame.length > 6 && (frame[6] as bool); // 止血:非法帧(宽高/平面为空)直接丢弃并上报诊断, // 避免下游组件越界(RGBA patch 后插件偶发 w/h=0 帧) if (width <= 0 || height <= 0 || planes.isEmpty || planes[0].isEmpty) { mainPort.send([ 'error', 'bad frame w=$width h=$height planes=${planes.length} ' 'p0=${planes.isNotEmpty ? planes[0].length : 0} ' 'stride=${strides.isNotEmpty ? strides[0] : '-'} ' 'bgra=$isBgra' ]); break; } final sw = Stopwatch()..start(); // 帧内容统计(诊断):Y/RGBA 平面 min/max/mean + 与上帧的平均绝对差。 // 均匀灰帧 → min≈max≈mean;静止灰帧 → diff≈0;真实画面 → 分布宽且 diff>0 final yPlane = planes[0]; var yMin = 255, yMax = 0, ySum = 0, diff = 0, sampled = 0; final prev = prevY; for (var i = 0; i < yPlane.length; i += 8) { final v = yPlane[i]; if (v < yMin) yMin = v; if (v > yMax) yMax = v; ySum += v; if (prev != null && i < prev.length) { final d = v - prev[i]; diff += d < 0 ? -d : d; } sampled++; } prevY = yPlane; final yMean = ySum / sampled; final yDiff = prev == null ? -1.0 : diff / (sampled * 255.0); // UV 平面统计(诊断):色序/值域异常会导致解码偏色 var uv1 = '-'; if (planes.length > 1) { final u = planes[1]; var uMin = 255, uMax = 0, uSum = 0, n = 0; for (var i = 0; i < u.length; i += 8) { final v = u[i]; if (v < uMin) uMin = v; if (v > uMax) uMax = v; uSum += v; n++; } uv1 = 's=${strides[1]} min=$uMin max=$uMax mean=${(uSum / n).toStringAsFixed(0)}'; if (planes.length > 2) { final v2 = planes[2]; var vMin = 255, vMax = 0, vSum = 0, n2 = 0; for (var i = 0; i < v2.length; i += 8) { final v = v2[i]; if (v < vMin) vMin = v; if (v > vMax) vMax = v; vSum += v; n2++; } uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}'; } } // 首帧(或相机重启后)自适应判定 YUV 值域与色序,再跑正式推理 if (!isBgra && !d.yuvModeKnown) { d.decideYuvChroma( planes: planes, strides: strides, width: width, height: height, ); } var results = d.detectRaw( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder, ); // 自愈:判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定 // (首帧模糊/暗帧导致启发式猜错时,画面稳定后 oracle 可分胜负) if (!isBgra && d.yuvModeKnown && !d.yuvRetried && results.length <= 1 && DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs > 1500 && d.retryDecision( planes: planes, strides: strides, width: width, height: height, )) { results = d.detectRaw( planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, ); } // 低分野鸡框过视觉先验(颜色/位置),减少户外误报 results = VisualPrior.filter( results, planes: planes, strides: strides, width: width, height: height, isBgra: isBgra, rgbaOrder: rgbaOrder, ); final motionRegions = m.detectMotionRaw( planes[0], strides[0], width, height); final noveltyRegions = b.updateRaw(planes[0], strides[0], width, height); sw.stop(); var dualDiag = ''; if (isBgra && DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) { lastDualMs = DateTime.now().millisecondsSinceEpoch; final a = d.diagnoseOrder( planes: planes, strides: strides, width: width, height: height, isBgra: true, rgbaOrder: false); final b = d.diagnoseOrder( planes: planes, strides: strides, width: width, height: height, isBgra: true, rgbaOrder: true); dualDiag = ' | dual BGRA:${a.$1}@${(a.$2 * 100).toStringAsFixed(1)}%' ' RGBA:${b.$1}@${(b.$2 * 100).toStringAsFixed(1)}%'; } mainPort.send([ 'result', rotation, width, height, results .map((r) => [r.label, r.score, r.left, r.top, r.right, r.bottom]) .toList(), motionRegions .map((mr) => [mr.left, mr.top, mr.right, mr.bottom]) .toList(), noveltyRegions .map((mr) => [mr.left, mr.top, mr.right, mr.bottom]) .toList(), sw.elapsedMilliseconds, 'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} ' 'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} ' 'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} ' 'uv1:[$uv1] | ${d.yuvDiag}$dualDiag', ]); break; case 'reset': motion?.reset(); background?.reset(); detector?.resetYuvMode(); break; case 'set-min-score': detector?.minScore = (list[1] as num).toDouble(); mainPort.send(['log', 'min-score=${detector?.minScore}']); } } catch (e, st) { mainPort.send([ 'error', '$e\n${st.toString().split('\n').take(3).join('\n')}' ]); } } }