迁移 Flutter 端与训练脚本,模型/训练产物移出 git(遵循纯代码约定)
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
import 'dart:math' as math;
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'detection_result.dart';
|
||||
import 'motion_aggregator.dart';
|
||||
|
||||
/// 静态场景背景建模:运行均值 + 方差,帧差高于自适应阈值的像素记为"新出现",
|
||||
/// 分块聚合为新颖区域(novelty)。
|
||||
///
|
||||
/// 固定机位下,常驻物体(键盘/石头/文字)永远属于背景、不产生新颖区域;
|
||||
/// 走进画面的目标(野鸡移动/新出现)才会触发。比相邻帧差分更强的证据:
|
||||
/// 风吹草动是持续的背景更新,不会长期标记为新颖。
|
||||
class BackgroundModel {
|
||||
final int maxWidth;
|
||||
final int maxHeight;
|
||||
|
||||
static const double learnRate = 0.05;
|
||||
static const double kSigma = 2.5;
|
||||
static const int minDiff = 15;
|
||||
static const int minPixels = 12;
|
||||
|
||||
Float32List? _mean;
|
||||
Float32List? _var;
|
||||
int _tw = 0;
|
||||
|
||||
BackgroundModel({this.maxWidth = 128, this.maxHeight = 128});
|
||||
|
||||
/// 后台 isolate 用原始数据接口(与 MotionDetector 同源:直接取 planes[0])。
|
||||
List<MotionRegion> updateRaw(
|
||||
Uint8List yPlane, int yStride, int width, int height) {
|
||||
final scale =
|
||||
maxWidth / width < maxHeight / height ? maxWidth / width : maxHeight / height;
|
||||
final tw = (width * scale).toInt().clamp(1, maxWidth);
|
||||
final th = (height * scale).toInt().clamp(1, maxHeight);
|
||||
if (tw == 0 || th == 0) return const [];
|
||||
|
||||
final gray = Float32List(tw * th);
|
||||
for (var oy = 0; oy < th; oy++) {
|
||||
final sy = (oy / scale).toInt().clamp(0, height - 1);
|
||||
final idx = oy * tw;
|
||||
for (var ox = 0; ox < tw; ox++) {
|
||||
final sx = (ox / scale).toInt().clamp(0, width - 1);
|
||||
gray[idx + ox] = yPlane[sy * yStride + sx].toDouble();
|
||||
}
|
||||
}
|
||||
return update(gray, tw, th);
|
||||
}
|
||||
|
||||
List<MotionRegion> update(Float32List gray, int tw, int th) {
|
||||
final n = gray.length;
|
||||
final mean = _mean;
|
||||
final variance = _var;
|
||||
if (mean == null || variance == null || mean.length != n || _tw != tw) {
|
||||
_mean = Float32List.fromList(gray);
|
||||
_var = Float32List(n);
|
||||
_tw = tw;
|
||||
return const [];
|
||||
}
|
||||
|
||||
final diff = Uint8List(n);
|
||||
var fgCount = 0;
|
||||
for (var i = 0; i < n; i++) {
|
||||
final g = gray[i];
|
||||
final m = mean[i];
|
||||
final d = (g - m).abs();
|
||||
if (d > kSigma * math.sqrt(variance[i]) + minDiff) {
|
||||
diff[i] = 1;
|
||||
fgCount++;
|
||||
// 前景像素不更新背景,避免把移动目标吸收进背景
|
||||
} else {
|
||||
// 静态像素缓慢吸收进背景,适应光照漂移
|
||||
final nm = m + learnRate * (g - m);
|
||||
mean[i] = nm;
|
||||
variance[i] =
|
||||
variance[i] + learnRate * ((g - nm) * (g - nm) - variance[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// 全屏大变化 → 相机移动/场景切换,重建背景
|
||||
if (fgCount > n ~/ 2) {
|
||||
_mean = null;
|
||||
_var = null;
|
||||
return const [];
|
||||
}
|
||||
if (fgCount < minPixels) return const [];
|
||||
return MotionAggregator.aggregate(diff, tw, th);
|
||||
}
|
||||
|
||||
/// 相机切换后重置,避免旧场景背景
|
||||
void reset() {
|
||||
_mean = null;
|
||||
_var = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
class ViewRect {
|
||||
final double left;
|
||||
final double top;
|
||||
final double right;
|
||||
final double bottom;
|
||||
|
||||
const ViewRect(this.left, this.top, this.right, this.bottom);
|
||||
|
||||
double get width => right - left;
|
||||
double get height => bottom - top;
|
||||
double get centerX => (left + right) / 2;
|
||||
double get centerY => (top + bottom) / 2;
|
||||
}
|
||||
|
||||
/// 模型归一化坐标 → 预览视图坐标(含传感器旋转与 FIT_CENTER 裁剪)。
|
||||
class CoordinateMapper {
|
||||
static ViewRect mapToView(
|
||||
double normLeft,
|
||||
double normTop,
|
||||
double normRight,
|
||||
double normBottom,
|
||||
int rotation,
|
||||
int imageW,
|
||||
int imageH,
|
||||
double viewW,
|
||||
double viewH,
|
||||
) {
|
||||
// 1) 旋转校正:图像方向 → 竖屏视图方向(归一化坐标)
|
||||
late final double x0, y0, x1, y1;
|
||||
switch (rotation) {
|
||||
case 90:
|
||||
x0 = 1 - normBottom;
|
||||
y0 = normLeft;
|
||||
x1 = 1 - normTop;
|
||||
y1 = normRight;
|
||||
case 180:
|
||||
x0 = 1 - normRight;
|
||||
y0 = 1 - normBottom;
|
||||
x1 = 1 - normLeft;
|
||||
y1 = 1 - normTop;
|
||||
case 270:
|
||||
x0 = normTop;
|
||||
y0 = 1 - normRight;
|
||||
x1 = normBottom;
|
||||
y1 = 1 - normLeft;
|
||||
default:
|
||||
x0 = normLeft;
|
||||
y0 = normTop;
|
||||
x1 = normRight;
|
||||
y1 = normBottom;
|
||||
}
|
||||
// 2) 旋转后图像在竖屏方向上的尺寸
|
||||
final portrait = rotation == 90 || rotation == 270;
|
||||
final portW = portrait ? imageH : imageW;
|
||||
final portH = portrait ? imageW : imageH;
|
||||
// 3) FIT_CENTER 缩放与居中偏移
|
||||
final scale = viewW / portW < viewH / portH
|
||||
? viewW / portW
|
||||
: viewH / portH;
|
||||
final offsetX = (viewW - portW * scale) / 2;
|
||||
final offsetY = (viewH - portH * scale) / 2;
|
||||
return ViewRect(
|
||||
x0 * portW * scale + offsetX,
|
||||
y0 * portH * scale + offsetY,
|
||||
x1 * portW * scale + offsetX,
|
||||
y1 * portH * scale + offsetY,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
class DetectionResult {
|
||||
final String label;
|
||||
final double score;
|
||||
final double left;
|
||||
final double top;
|
||||
final double right;
|
||||
final double bottom;
|
||||
|
||||
/// 轨迹已确认(多帧稳定/高分/活动确认),false = 候选,渲染为虚线
|
||||
final bool confirmed;
|
||||
|
||||
const DetectionResult({
|
||||
required this.label,
|
||||
required this.score,
|
||||
required this.left,
|
||||
required this.top,
|
||||
required this.right,
|
||||
required this.bottom,
|
||||
this.confirmed = true,
|
||||
});
|
||||
|
||||
double get width => right - left;
|
||||
double get height => bottom - top;
|
||||
double get centerX => (left + right) / 2;
|
||||
double get centerY => (top + bottom) / 2;
|
||||
|
||||
DetectionResult copyWith({
|
||||
double? score,
|
||||
double? left,
|
||||
double? top,
|
||||
double? right,
|
||||
double? bottom,
|
||||
bool? confirmed,
|
||||
}) =>
|
||||
DetectionResult(
|
||||
label: label,
|
||||
score: score ?? this.score,
|
||||
left: left ?? this.left,
|
||||
top: top ?? this.top,
|
||||
right: right ?? this.right,
|
||||
bottom: bottom ?? this.bottom,
|
||||
confirmed: confirmed ?? this.confirmed,
|
||||
);
|
||||
}
|
||||
|
||||
class MotionRegion {
|
||||
final double left;
|
||||
final double top;
|
||||
final double right;
|
||||
final double bottom;
|
||||
|
||||
const MotionRegion(this.left, this.top, this.right, this.bottom);
|
||||
|
||||
double get centerX => (left + right) / 2;
|
||||
double get centerY => (top + bottom) / 2;
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import 'dart:async';
|
||||
import 'dart:isolate';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:camera/camera.dart';
|
||||
import 'package:flutter/foundation.dart' show debugPrint;
|
||||
import 'package:flutter/services.dart' show rootBundle;
|
||||
|
||||
import '../camera/motion_detector.dart';
|
||||
import 'background_model.dart';
|
||||
import 'detection_result.dart';
|
||||
import 'tflite_detector.dart';
|
||||
import 'visual_prior.dart';
|
||||
|
||||
/// 推理工作单元:模型加载与检测全部在后台 isolate 执行,
|
||||
/// 主 isolate 只投递帧数据、接收结果,UI 不被推理阻塞(iOS 真机卡顿根因)。
|
||||
class DetectorWorker {
|
||||
static const String modelAsset = 'assets/model.tflite';
|
||||
static const String labelsAsset = 'assets/labels.txt';
|
||||
|
||||
final Isolate _isolate;
|
||||
final ReceivePort _responses;
|
||||
|
||||
final _controlPort = Completer<SendPort>();
|
||||
final _ready = Completer<void>();
|
||||
|
||||
SendPort? _port;
|
||||
|
||||
/// 在途帧数(主 isolate 侧计数,用于丢帧)
|
||||
int _inFlight = 0;
|
||||
bool _dead = false;
|
||||
|
||||
/// 结果回调:结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 / 处理耗时 ms
|
||||
void Function(List<DetectionResult>, List<MotionRegion>, List<MotionRegion>,
|
||||
int, int, int, int)? onResult;
|
||||
|
||||
/// 单帧处理异常回调(不影响相机流)
|
||||
void Function(String)? onError;
|
||||
|
||||
/// 最近一次创建失败的诊断原因(UI 展示用)
|
||||
static String? lastLoadError;
|
||||
|
||||
/// worker 最近上报的执行步骤(诊断用)
|
||||
static String? lastLog;
|
||||
|
||||
DetectorWorker._(this._isolate, this._responses) {
|
||||
_responses.listen(_onMessage, onDone: () {
|
||||
_dead = true;
|
||||
if (!_ready.isCompleted) {
|
||||
_ready.completeError(StateError('推理进程异常退出'));
|
||||
}
|
||||
onError?.call('推理进程异常退出');
|
||||
});
|
||||
}
|
||||
|
||||
/// 读取模型资产并启动后台推理 isolate;加载失败返回 null(App 降级为仅预览)。
|
||||
static Future<DetectorWorker?> create() async {
|
||||
try {
|
||||
final data = await rootBundle.load(modelAsset);
|
||||
final modelBytes =
|
||||
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes);
|
||||
final labels = (await rootBundle.loadString(labelsAsset))
|
||||
.split('\n')
|
||||
.where((l) => l.trim().isNotEmpty)
|
||||
.toList();
|
||||
|
||||
final responses = ReceivePort();
|
||||
final isolate = await Isolate.spawn(_workerMain, responses.sendPort);
|
||||
final worker = DetectorWorker._(isolate, responses);
|
||||
|
||||
final port = await worker._controlPort.future
|
||||
.timeout(const Duration(seconds: 10),
|
||||
onTimeout: () => throw TimeoutException('worker port timeout'));
|
||||
worker._port = port;
|
||||
port.send(['load', modelBytes, labels]);
|
||||
await worker._ready.future
|
||||
.timeout(const Duration(seconds: 20), onTimeout: () {
|
||||
throw TimeoutException('model load timeout');
|
||||
});
|
||||
return worker;
|
||||
} catch (e) {
|
||||
lastLoadError = e.toString();
|
||||
debugPrint('[DetectorWorker] create failed: $e');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// 是否忙(上一帧尚未返回):忙则丢帧,避免在途积压
|
||||
bool get busy => _inFlight > 0;
|
||||
|
||||
void analyze(CameraImage image, int rotationDegrees) {
|
||||
final port = _port;
|
||||
if (port == null || _dead) return;
|
||||
_inFlight++;
|
||||
port.send([
|
||||
'frame',
|
||||
[
|
||||
image.planes.map((p) => p.bytes).toList(),
|
||||
image.planes.map((p) => p.bytesPerRow).toList(),
|
||||
image.width,
|
||||
image.height,
|
||||
image.format.group == ImageFormatGroup.bgra8888,
|
||||
rotationDegrees,
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
void _onMessage(dynamic msg) {
|
||||
final list = msg as List;
|
||||
switch (list[0] as String) {
|
||||
case 'port':
|
||||
_controlPort.complete(list[1] as SendPort);
|
||||
break;
|
||||
case 'ready':
|
||||
_ready.complete();
|
||||
break;
|
||||
case 'load-error':
|
||||
_ready.completeError(StateError(
|
||||
list.length > 1 ? list[1] as String : 'model load failed'));
|
||||
break;
|
||||
case 'result':
|
||||
_inFlight--;
|
||||
final dets = (list[4] as List).map((d) {
|
||||
final v = d as List;
|
||||
return DetectionResult(
|
||||
label: v[0] as String,
|
||||
score: v[1] as double,
|
||||
left: v[2] as double,
|
||||
top: v[3] as double,
|
||||
right: v[4] as double,
|
||||
bottom: v[5] as double,
|
||||
);
|
||||
}).toList();
|
||||
final motion = (list[5] as List)
|
||||
.map((m) => m as List)
|
||||
.map((v) => MotionRegion(
|
||||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||||
.toList();
|
||||
final novelty = (list[6] as List)
|
||||
.map((m) => m as List)
|
||||
.map((v) => MotionRegion(
|
||||
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
|
||||
.toList();
|
||||
onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int,
|
||||
list[3] as int, list[7] as int);
|
||||
break;
|
||||
case 'log':
|
||||
lastLog = list[1] as String;
|
||||
debugPrint('[DetectorWorker] $lastLog');
|
||||
break;
|
||||
case 'error':
|
||||
_inFlight--;
|
||||
onError?.call(list[1] as String);
|
||||
}
|
||||
}
|
||||
|
||||
/// 相机切换/场景变化后重置运动与背景参考
|
||||
void reset() {
|
||||
final port = _port;
|
||||
if (port == null || _dead) return;
|
||||
port.send(['reset']);
|
||||
}
|
||||
|
||||
void dispose() {
|
||||
_dead = true;
|
||||
_isolate.kill(priority: Isolate.immediate);
|
||||
_responses.close();
|
||||
}
|
||||
}
|
||||
|
||||
/// 后台 isolate 入口:串行处理 load / frame / reset 命令。
|
||||
/// 所有回发必须走 [mainPort](主 isolate 的端口);control 是 worker 自己的
|
||||
/// 收件箱,往 control.sendPort 发消息等于发给自己,主 isolate 永远收不到。
|
||||
Future<void> _workerMain(SendPort mainPort) async {
|
||||
final control = ReceivePort();
|
||||
mainPort.send(['port', control.sendPort]);
|
||||
mainPort.send(['log', 'worker-start']);
|
||||
|
||||
TfliteDetector? detector;
|
||||
MotionDetector? motion;
|
||||
BackgroundModel? background;
|
||||
await for (final msg in control) {
|
||||
try {
|
||||
final list = msg as List;
|
||||
switch (list[0] as String) {
|
||||
case 'load':
|
||||
mainPort.send(['log', 'load-received']);
|
||||
try {
|
||||
detector = await TfliteDetector.fromBuffer(
|
||||
list[1] as Uint8List, (list[2] as List).cast<String>());
|
||||
if (detector == null) {
|
||||
mainPort.send(['load-error', 'fromBuffer 返回 null']);
|
||||
} else {
|
||||
mainPort.send(['log', 'fromBuffer-ok']);
|
||||
motion = MotionDetector();
|
||||
background = BackgroundModel();
|
||||
mainPort.send(['ready']);
|
||||
}
|
||||
} catch (e) {
|
||||
mainPort.send(['load-error', '$e']);
|
||||
}
|
||||
break;
|
||||
case 'frame':
|
||||
final d = detector;
|
||||
final m = motion;
|
||||
final b = background;
|
||||
if (d == null || m == null || b == null) break;
|
||||
final frame = list[1] as List;
|
||||
final planes = (frame[0] as List).cast<Uint8List>();
|
||||
final strides = (frame[1] as List).cast<int>();
|
||||
final width = frame[2] as int;
|
||||
final height = frame[3] as int;
|
||||
final isBgra = frame[4] as bool;
|
||||
final rotation = frame[5] as int;
|
||||
|
||||
final sw = Stopwatch()..start();
|
||||
var results = d.detectRaw(
|
||||
planes: planes,
|
||||
strides: strides,
|
||||
width: width,
|
||||
height: height,
|
||||
isBgra: isBgra,
|
||||
);
|
||||
// 低分野鸡框过视觉先验(颜色/位置),减少户外误报
|
||||
results = VisualPrior.filter(
|
||||
results,
|
||||
planes: planes,
|
||||
strides: strides,
|
||||
width: width,
|
||||
height: height,
|
||||
isBgra: isBgra,
|
||||
);
|
||||
final motionRegions = m.detectMotionRaw(
|
||||
planes[0], strides[0], width, height);
|
||||
final noveltyRegions =
|
||||
b.updateRaw(planes[0], strides[0], width, height);
|
||||
sw.stop();
|
||||
|
||||
mainPort.send([
|
||||
'result',
|
||||
rotation,
|
||||
width,
|
||||
height,
|
||||
results
|
||||
.map((r) =>
|
||||
[r.label, r.score, r.left, r.top, r.right, r.bottom])
|
||||
.toList(),
|
||||
motionRegions
|
||||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||||
.toList(),
|
||||
noveltyRegions
|
||||
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
|
||||
.toList(),
|
||||
sw.elapsedMilliseconds,
|
||||
]);
|
||||
break;
|
||||
case 'reset':
|
||||
motion?.reset();
|
||||
background?.reset();
|
||||
}
|
||||
} catch (e) {
|
||||
mainPort.send(['error', '$e']);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import 'dart:math' as math;
|
||||
|
||||
import 'detection_result.dart';
|
||||
|
||||
/// 帧差运动聚合:每像素 0/1 差分掩码 → 8x8 分块统计 → 连通块聚合为运动区域。
|
||||
class MotionAggregator {
|
||||
static const int blockGrid = 8;
|
||||
static const double blockActiveRatio = 0.30;
|
||||
static const int maxRegions = 3;
|
||||
static const int diffThreshold = 25;
|
||||
|
||||
static List<MotionRegion> aggregate(List<int> diff, int width, int height) {
|
||||
final bw = width ~/ blockGrid;
|
||||
final bh = height ~/ blockGrid;
|
||||
if (bw == 0 || bh == 0) return const [];
|
||||
|
||||
final active = List<bool>.filled(blockGrid * blockGrid, false);
|
||||
for (var by = 0; by < blockGrid; by++) {
|
||||
for (var bx = 0; bx < blockGrid; bx++) {
|
||||
final blockW = bx == blockGrid - 1 ? width - bx * bw : bw;
|
||||
final blockH = by == blockGrid - 1 ? height - by * bh : bh;
|
||||
var motion = 0;
|
||||
for (var y = by * bh; y < by * bh + blockH; y++) {
|
||||
var idx = y * width + bx * bw;
|
||||
for (var x = 0; x < blockW; x++) {
|
||||
motion += diff[idx + x];
|
||||
}
|
||||
idx += width;
|
||||
}
|
||||
active[by * blockGrid + bx] =
|
||||
motion > blockW * blockH * blockActiveRatio;
|
||||
}
|
||||
}
|
||||
|
||||
final regions = <MotionRegion>[];
|
||||
final visited = List<bool>.filled(active.length, false);
|
||||
for (var i = 0; i < active.length; i++) {
|
||||
if (!active[i] || visited[i]) continue;
|
||||
var minX = blockGrid, minY = blockGrid, maxX = -1, maxY = -1;
|
||||
final stack = <int>[i];
|
||||
visited[i] = true;
|
||||
while (stack.isNotEmpty) {
|
||||
final cur = stack.removeLast();
|
||||
final bx = cur % blockGrid;
|
||||
final by = cur ~/ blockGrid;
|
||||
if (bx < minX) minX = bx;
|
||||
if (bx > maxX) maxX = bx;
|
||||
if (by < minY) minY = by;
|
||||
if (by > maxY) maxY = by;
|
||||
for (final nb in neighbors(cur)) {
|
||||
if (active[nb] && !visited[nb]) {
|
||||
visited[nb] = true;
|
||||
stack.add(nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (maxX - minX > 3 || maxY - minY > 3) continue; // 全屏噪声过滤
|
||||
regions.add(MotionRegion(
|
||||
minX * bw / width,
|
||||
minY * bh / height,
|
||||
math.min((maxX + 1) * bw, width) / width,
|
||||
math.min((maxY + 1) * bh, height) / height,
|
||||
));
|
||||
if (regions.length >= maxRegions) break;
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
|
||||
static List<int> neighbors(int i) {
|
||||
final bx = i % blockGrid;
|
||||
final by = i ~/ blockGrid;
|
||||
final list = <int>[];
|
||||
if (bx > 0) list.add(i - 1);
|
||||
if (bx < blockGrid - 1) list.add(i + 1);
|
||||
if (by > 0) list.add(i - blockGrid);
|
||||
if (by < blockGrid - 1) list.add(i + blockGrid);
|
||||
return list;
|
||||
}
|
||||
|
||||
/// 检测框中心是否落在运动区域内(用于置信度提升判定)
|
||||
static bool centerInRegion(DetectionResult box, MotionRegion region) =>
|
||||
box.centerX >= region.left &&
|
||||
box.centerX <= region.right &&
|
||||
box.centerY >= region.top &&
|
||||
box.centerY <= region.bottom;
|
||||
|
||||
/// 帧差掩码:|g - prev| > threshold → 1
|
||||
static List<int> diffMask(List<int> gray, List<int> prev,
|
||||
[int threshold = diffThreshold]) {
|
||||
final diff = List<int>.filled(gray.length, 0);
|
||||
for (var i = 0; i < gray.length; i++) {
|
||||
diff[i] = (gray[i] - prev[i]).abs() > threshold ? 1 : 0;
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import 'detection_result.dart';
|
||||
|
||||
double iou(DetectionResult a, DetectionResult b) {
|
||||
final x0 = a.left > b.left ? a.left : b.left;
|
||||
final y0 = a.top > b.top ? a.top : b.top;
|
||||
final x1 = a.right < b.right ? a.right : b.right;
|
||||
final y1 = a.bottom < b.bottom ? a.bottom : b.bottom;
|
||||
if (x1 <= x0 || y1 <= y0) return 0;
|
||||
final inter = (x1 - x0) * (y1 - y0);
|
||||
final union = a.width * a.height + b.width * b.height - inter;
|
||||
return union <= 0 ? 0 : inter / union;
|
||||
}
|
||||
|
||||
List<DetectionResult> nms(List<DetectionResult> boxes, double iouThreshold) {
|
||||
final sorted = [...boxes]..sort((a, b) => b.score.compareTo(a.score));
|
||||
final kept = <DetectionResult>[];
|
||||
for (final b in sorted) {
|
||||
if (!kept.any((k) => iou(b, k) > iouThreshold)) kept.add(b);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:tflite_flutter/tflite_flutter.dart';
|
||||
|
||||
import 'detection_result.dart';
|
||||
import 'nms.dart';
|
||||
|
||||
/// YOLOv8n 端侧推理实现(对应 Kotlin TFLiteDetector)。
|
||||
/// 模型输出布局(ultralytics litert 导出):[1, 4 + nc, anchors],
|
||||
/// cx/cy/w/h 已归一化,类别得分已过 sigmoid;按 out[dim][anchor] 索引。
|
||||
/// 输入为 NCHW [1, 3, 704, 704](litert 导出保留 torch 布局)。
|
||||
class TfliteDetector {
|
||||
static const int inputSize = 704;
|
||||
// 野鸡数据置信度普遍偏低(0.1~0.2 量级),保留低分池供运动检测提升
|
||||
static const double minScore = 0.10;
|
||||
static const double iouThreshold = 0.45;
|
||||
static const int maxDetections = 20;
|
||||
static const String modelAsset = 'assets/model.tflite';
|
||||
static const String labelsAsset = 'assets/labels.txt';
|
||||
|
||||
final Interpreter _interpreter;
|
||||
final List<String> _labels;
|
||||
final int _numClasses;
|
||||
final int _numAnchors;
|
||||
|
||||
final Float32List _input =
|
||||
Float32List(1 * inputSize * inputSize * 3);
|
||||
|
||||
/// 输出按模型形状 [1, 4+nc, anchors] 的嵌套 List 组织,
|
||||
/// run() 要求输出对象形状与模型完全一致(扁平 List 会被拒)。
|
||||
final List<List<List<double>>> _output;
|
||||
|
||||
TfliteDetector._(this._interpreter, this._labels, this._numClasses,
|
||||
this._numAnchors, this._output);
|
||||
|
||||
/// 模型缺失或加载失败返回 null(App 降级为仅预览)。
|
||||
/// 在后台 isolate 内调用(模型字节由主 isolate 读取后传入)。
|
||||
static Future<TfliteDetector?> fromBuffer(
|
||||
Uint8List bytes, List<String> labels) async {
|
||||
try {
|
||||
final interpreter = Interpreter.fromBuffer(
|
||||
bytes,
|
||||
options: InterpreterOptions()..threads = 4,
|
||||
);
|
||||
return TfliteDetector._fromModel(interpreter, labels);
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// 输出布局 [1, 4+nc, anchors] 取自模型本身,类别数不与 labels 文件长度耦合。
|
||||
factory TfliteDetector._fromModel(
|
||||
Interpreter interpreter, List<String> labels) {
|
||||
final shape = interpreter.getOutputTensor(0).shape;
|
||||
final numClasses =
|
||||
shape.length >= 3 && shape[1] > 4 ? shape[1] - 4 : labels.length;
|
||||
final numAnchors = shape.length >= 3 && shape[2] > 0 ? shape[2] : 2100;
|
||||
final output = List.generate(
|
||||
1,
|
||||
(_) => List.generate(
|
||||
numClasses + 4,
|
||||
(_) => List<double>.filled(numAnchors, 0),
|
||||
),
|
||||
);
|
||||
return TfliteDetector._(
|
||||
interpreter, labels, numClasses, numAnchors, output);
|
||||
}
|
||||
|
||||
/// 原始数据接口(后台 isolate 用,不依赖 CameraImage)。
|
||||
/// 输出坐标统一反算为原图归一化空间(与 MotionDetector 一致),
|
||||
/// 否则 CENTER_CROP 裁剪偏移会让检测框系统性偏移。
|
||||
List<DetectionResult> detectRaw({
|
||||
required List<Uint8List> planes,
|
||||
required List<int> strides,
|
||||
required int width,
|
||||
required int height,
|
||||
required bool isBgra,
|
||||
}) {
|
||||
preprocess(
|
||||
planes: planes,
|
||||
strides: strides,
|
||||
width: width,
|
||||
height: height,
|
||||
isBgra: isBgra);
|
||||
// 传原始字节视图而非 Float32List:tflite_flutter 会对非 ByteBuffer/Uint8List
|
||||
// 输入调用 resizeInputTensor(1 维 [1486848]),使 node 0 TRANSPOSE prepare 失败
|
||||
_interpreter.run(_input.buffer.asUint8List(), _output);
|
||||
final dets = postprocess();
|
||||
// 反算与 preprocess 的 scale/dx/dy 公式一致(704 输入空间 → 原图归一化)
|
||||
final scale = inputSize / width < inputSize / height
|
||||
? inputSize / width
|
||||
: inputSize / height;
|
||||
final dx = (inputSize - width * scale) / 2;
|
||||
final dy = (inputSize - height * scale) / 2;
|
||||
if (dx == 0 && dy == 0) return dets;
|
||||
return dets
|
||||
.map((r) => r.copyWith(
|
||||
left: (r.left * inputSize - dx) / (width * scale),
|
||||
right: (r.right * inputSize - dx) / (width * scale),
|
||||
top: (r.top * inputSize - dy) / (height * scale),
|
||||
bottom: (r.bottom * inputSize - dy) / (height * scale),
|
||||
))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/// 按像素格式分派:iOS bgra8888 单平面 / Android yuv420 多平面。
|
||||
void preprocess({
|
||||
required List<Uint8List> planes,
|
||||
required List<int> strides,
|
||||
required int width,
|
||||
required int height,
|
||||
required bool isBgra,
|
||||
}) {
|
||||
if (isBgra) {
|
||||
_preprocessBgra(planes[0], strides[0], width, height);
|
||||
} else {
|
||||
_preprocessYuv(planes, strides, width, height);
|
||||
}
|
||||
}
|
||||
|
||||
/// BGRA8888 单平面(iOS):每像素 4 字节 [b,g,r,a],双线性采样,
|
||||
/// letterbox(等比缩到长边 704,短边黑边补 0,与 YOLO 训练一致)。
|
||||
void _preprocessBgra(Uint8List src, int stride, int srcW, int srcH) {
|
||||
final plane = inputSize * inputSize;
|
||||
final scale = inputSize / srcW < inputSize / srcH
|
||||
? inputSize / srcW
|
||||
: inputSize / srcH;
|
||||
final dx = (inputSize - srcW * scale) / 2;
|
||||
final dy = (inputSize - srcH * scale) / 2;
|
||||
|
||||
for (var oy = 0; oy < inputSize; oy++) {
|
||||
final syf = (oy - dy) / scale;
|
||||
if (syf < 0 || syf >= srcH) {
|
||||
for (var ox = 0; ox < inputSize; ox++) {
|
||||
final p = oy * inputSize + ox;
|
||||
_input[p] = 0;
|
||||
_input[p + plane] = 0;
|
||||
_input[p + 2 * plane] = 0;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (var ox = 0; ox < inputSize; ox++) {
|
||||
final p = oy * inputSize + ox;
|
||||
final sxf = (ox - dx) / scale;
|
||||
if (sxf < 0 || sxf >= srcW) {
|
||||
_input[p] = 0;
|
||||
_input[p + plane] = 0;
|
||||
_input[p + 2 * plane] = 0;
|
||||
continue;
|
||||
}
|
||||
final x0 = sxf.floor(), y0 = syf.floor();
|
||||
final x1 = x0 < srcW - 1 ? x0 + 1 : x0;
|
||||
final y1 = y0 < srcH - 1 ? y0 + 1 : y0;
|
||||
final fx = sxf - x0, fy = syf - y0;
|
||||
|
||||
// BGRA 字节序:+0 B、+1 G、+2 R、+3 A
|
||||
final i00 = y0 * stride + x0 * 4;
|
||||
final i10 = y0 * stride + x1 * 4;
|
||||
final i01 = y1 * stride + x0 * 4;
|
||||
final i11 = y1 * stride + x1 * 4;
|
||||
final r00 = src[i00 + 2].toDouble();
|
||||
final g00 = src[i00 + 1].toDouble();
|
||||
final b00 = src[i00].toDouble();
|
||||
final r10 = src[i10 + 2].toDouble();
|
||||
final g10 = src[i10 + 1].toDouble();
|
||||
final b10 = src[i10].toDouble();
|
||||
final r01 = src[i01 + 2].toDouble();
|
||||
final g01 = src[i01 + 1].toDouble();
|
||||
final b01 = src[i01].toDouble();
|
||||
final r11 = src[i11 + 2].toDouble();
|
||||
final g11 = src[i11 + 1].toDouble();
|
||||
final b11 = src[i11].toDouble();
|
||||
|
||||
_input[p] = _bl(r00, r10, r01, r11, fx, fy) / 255.0;
|
||||
_input[p + plane] = _bl(g00, g10, g01, g11, fx, fy) / 255.0;
|
||||
_input[p + 2 * plane] = _bl(b00, b10, b01, b11, fx, fy) / 255.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// letterbox 缩放 + YUV → RGB 归一化 0~1(NCHW),双线性采样。
|
||||
/// 兼容 NV12(iOS 双平面,UV 交错)与 I420(Android 三平面)。
|
||||
void _preprocessYuv(
|
||||
List<Uint8List> planes, List<int> strides, int srcW, int srcH) {
|
||||
final plane = inputSize * inputSize;
|
||||
final y = planes[0];
|
||||
final nv12 = planes.length == 2;
|
||||
final uv = nv12 ? planes[1] : null;
|
||||
final u = nv12 ? null : planes[1];
|
||||
final v = nv12 ? null : planes[2];
|
||||
final yStride = strides[0];
|
||||
final uvStride = strides[1];
|
||||
|
||||
// U/V 平面采样(nv12:偶位 U 奇位 V;i420:三平面分离)
|
||||
double uAt(int x, int y) => nv12
|
||||
? uv![y * uvStride + x * 2] - 128.0
|
||||
: u![y * uvStride + x] - 128.0;
|
||||
double vAt(int x, int y) => nv12
|
||||
? uv![y * uvStride + x * 2 + 1] - 128.0
|
||||
: v![y * uvStride + x] - 128.0;
|
||||
|
||||
final scale = inputSize / srcW < inputSize / srcH
|
||||
? inputSize / srcW
|
||||
: inputSize / srcH;
|
||||
final dx = (inputSize - srcW * scale) / 2;
|
||||
final dy = (inputSize - srcH * scale) / 2;
|
||||
|
||||
for (var oy = 0; oy < inputSize; oy++) {
|
||||
final syf = (oy - dy) / scale;
|
||||
if (syf < 0 || syf >= srcH) {
|
||||
for (var ox = 0; ox < inputSize; ox++) {
|
||||
final p = oy * inputSize + ox;
|
||||
_input[p] = 0;
|
||||
_input[p + plane] = 0;
|
||||
_input[p + 2 * plane] = 0;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (var ox = 0; ox < inputSize; ox++) {
|
||||
final p = oy * inputSize + ox;
|
||||
final sxf = (ox - dx) / scale;
|
||||
if (sxf < 0 || sxf >= srcW) {
|
||||
_input[p] = 0;
|
||||
_input[p + plane] = 0;
|
||||
_input[p + 2 * plane] = 0;
|
||||
continue;
|
||||
}
|
||||
final x0 = sxf.floor(), y0 = syf.floor();
|
||||
final x1 = x0 < srcW - 1 ? x0 + 1 : x0;
|
||||
final y1 = y0 < srcH - 1 ? y0 + 1 : y0;
|
||||
final fx = sxf - x0, fy = syf - y0;
|
||||
|
||||
// Y 双线性
|
||||
final y00 = y[y0 * yStride + x0].toDouble();
|
||||
final y10 = y[y0 * yStride + x1].toDouble();
|
||||
final y01 = y[y1 * yStride + x0].toDouble();
|
||||
final y11 = y[y1 * yStride + x1].toDouble();
|
||||
final yy = _bl(y00, y10, y01, y11, fx, fy);
|
||||
|
||||
// U/V 双线性(4:2:0 半分辨率,按像素坐标定位后除 2)
|
||||
final maxUx = srcW ~/ 2 - 1;
|
||||
final maxUy = srcH ~/ 2 - 1;
|
||||
final ux0 = (x0 ~/ 2).clamp(0, maxUx).toInt();
|
||||
final uy0 = (y0 ~/ 2).clamp(0, maxUy).toInt();
|
||||
final ux1 = (x1 ~/ 2).clamp(0, maxUx).toInt();
|
||||
final uy1 = (y1 ~/ 2).clamp(0, maxUy).toInt();
|
||||
final u00 = uAt(ux0, uy0);
|
||||
final u10 = uAt(ux1, uy0);
|
||||
final u01 = uAt(ux0, uy1);
|
||||
final u11 = uAt(ux1, uy1);
|
||||
final uu = _bl(u00, u10, u01, u11, fx, fy);
|
||||
|
||||
final v00 = vAt(ux0, uy0);
|
||||
final v10 = vAt(ux1, uy0);
|
||||
final v01 = vAt(ux0, uy1);
|
||||
final v11 = vAt(ux1, uy1);
|
||||
final vv = _bl(v00, v10, v01, v11, fx, fy);
|
||||
|
||||
// 有限范围展开(VideoRange Y 16~235,Cb/Cr 16~240)
|
||||
final yr = (yy - 16.0) * (255.0 / 219.0);
|
||||
final un = uu * (255.0 / 224.0);
|
||||
final vn = vv * (255.0 / 224.0);
|
||||
|
||||
// NCHW:r/g/b 分平面存储
|
||||
_input[p] = (yr + 1.402 * vn) / 255.0;
|
||||
_input[p + plane] = (yr - 0.344136 * un - 0.714136 * vn) / 255.0;
|
||||
_input[p + 2 * plane] = (yr + 1.772 * un) / 255.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static double _bl(double a, double b, double c, double d, double fx,
|
||||
double fy) =>
|
||||
(1 - fx) * (1 - fy) * a + fx * (1 - fy) * b +
|
||||
(1 - fx) * fy * c + fx * fy * d;
|
||||
|
||||
List<DetectionResult> postprocess() {
|
||||
final out = _output[0];
|
||||
final boxes = <DetectionResult>[];
|
||||
for (var a = 0; a < _numAnchors; a++) {
|
||||
final cx = out[0][a];
|
||||
final cy = out[1][a];
|
||||
final w = out[2][a];
|
||||
final h = out[3][a];
|
||||
var bestCls = 0;
|
||||
var bestScore = 0.0;
|
||||
for (var c = 0; c < _numClasses; c++) {
|
||||
final s = out[4 + c][a];
|
||||
if (s > bestScore) {
|
||||
bestScore = s;
|
||||
bestCls = c;
|
||||
}
|
||||
}
|
||||
final label =
|
||||
bestCls < _labels.length ? _labels[bestCls] : 'unknown';
|
||||
// 低分池保留,供运动检测提升显示
|
||||
if (bestScore < minScore) continue;
|
||||
boxes.add(DetectionResult(
|
||||
label: label,
|
||||
score: bestScore,
|
||||
left: (cx - w / 2).clamp(0.0, 1.0),
|
||||
top: (cy - h / 2).clamp(0.0, 1.0),
|
||||
right: (cx + w / 2).clamp(0.0, 1.0),
|
||||
bottom: (cy + h / 2).clamp(0.0, 1.0),
|
||||
));
|
||||
}
|
||||
final kept = nms(boxes, iouThreshold);
|
||||
return kept.take(maxDetections).toList();
|
||||
}
|
||||
|
||||
void dispose() => _interpreter.close();
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import 'dart:math' as math;
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'detection_result.dart';
|
||||
|
||||
/// 运行时视觉先验:对低置信度野鸡框做多线索过滤,降低户外误报。
|
||||
///
|
||||
/// 仅对 score < [maxScore](0.35)的 pheasant 框生效;高分框与
|
||||
/// suspect(生境预警)不参与过滤,避免误杀。
|
||||
///
|
||||
/// 线索:
|
||||
/// - 颜色:绿色主导(草/叶)、蓝色主导(天空/水)、平坦低饱和(键盘/石头/文字)
|
||||
/// - 位置:中心在画面上部 15%(天空区)——野鸡是地栖动物,不会出现在天空
|
||||
///
|
||||
/// 采样在原始 planes 上进行(后台 isolate 内,不依赖 UI 线程)。
|
||||
class VisualPrior {
|
||||
static const double maxScore = 0.35;
|
||||
static const double skyTopRatio = 0.15;
|
||||
|
||||
// 颜色判定阈值(与 tflite_detector 的 YUV 有限范围展开一致)
|
||||
static const double greenDiff = 20;
|
||||
static const double blueDiff = 10;
|
||||
static const double flatRange = 10;
|
||||
|
||||
// 采样点中满足条件的比例超过即拒绝
|
||||
static const double greenRatio = 0.5;
|
||||
static const double blueRatio = 0.4;
|
||||
static const double flatRatio = 0.6;
|
||||
|
||||
static List<DetectionResult> filter(
|
||||
List<DetectionResult> results, {
|
||||
required List<Uint8List> planes,
|
||||
required List<int> strides,
|
||||
required int width,
|
||||
required int height,
|
||||
required bool isBgra,
|
||||
}) {
|
||||
if (results.isEmpty || width <= 0 || height <= 0) return results;
|
||||
final kept = <DetectionResult>[];
|
||||
for (final r in results) {
|
||||
final lowConfPheasant = r.label == 'pheasant' && r.score < maxScore;
|
||||
if (lowConfPheasant && _reject(r, planes, strides, width, height, isBgra)) {
|
||||
continue;
|
||||
}
|
||||
kept.add(r);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
static bool _reject(DetectionResult r, List<Uint8List> planes,
|
||||
List<int> strides, int width, int height, bool isBgra) {
|
||||
// 位置线索:detectRaw 输出为图像坐标系,centerY 直接可判天空区
|
||||
if (r.centerY < skyTopRatio) return true;
|
||||
|
||||
// 颜色线索:框中心 ±20% 区域 5×5 采样(小框采样点重合也没关系)
|
||||
final cx = (r.centerX * width).round().clamp(0, width - 1).toInt();
|
||||
final cy = (r.centerY * height).round().clamp(0, height - 1).toInt();
|
||||
final halfW = math.max(1.0, r.width * width * 0.2);
|
||||
final halfH = math.max(1.0, r.height * height * 0.2);
|
||||
|
||||
var green = 0, blue = 0, flat = 0, total = 0;
|
||||
for (var gy = -2; gy <= 2; gy++) {
|
||||
for (var gx = -2; gx <= 2; gx++) {
|
||||
final px = (cx + gx * halfW / 2).round().clamp(0, width - 1).toInt();
|
||||
final py = (cy + gy * halfH / 2).round().clamp(0, height - 1).toInt();
|
||||
final (r_, g_, b_) = _pixel(planes, strides, px, py, width, height, isBgra);
|
||||
total++;
|
||||
final mn = math.min(r_, math.min(g_, b_));
|
||||
final mx = math.max(r_, math.max(g_, b_));
|
||||
if (g_ - r_ > greenDiff && g_ - b_ > greenDiff) green++;
|
||||
if (b_ > r_ + blueDiff) blue++;
|
||||
if (mx - mn < flatRange) flat++;
|
||||
}
|
||||
}
|
||||
if (total == 0) return false;
|
||||
if (green / total > greenRatio) return true;
|
||||
if (blue / total > blueRatio) return true;
|
||||
if (flat / total > flatRatio) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// 读取单像素 RGB(0~255)。
|
||||
/// BGRA 单平面:每像素 4 字节 [b,g,r,a];
|
||||
/// YUV:y 平面 + 4:2:0 半分辨率 U/V(NV12 交错或 I420 分离)。
|
||||
static (double, double, double) _pixel(List<Uint8List> planes,
|
||||
List<int> strides, int x, int y, int width, int height, bool isBgra) {
|
||||
if (isBgra) {
|
||||
final src = planes[0];
|
||||
final i = y * strides[0] + x * 4;
|
||||
return (src[i + 2].toDouble(), src[i + 1].toDouble(), src[i].toDouble());
|
||||
}
|
||||
final yy =
|
||||
(planes[0][y * strides[0] + x] - 16.0) * (255.0 / 219.0);
|
||||
final nv12 = planes.length == 2;
|
||||
final ux = (x ~/ 2).clamp(0, width ~/ 2 - 1).toInt();
|
||||
final uy = (y ~/ 2).clamp(0, height ~/ 2 - 1).toInt();
|
||||
final uvStride = strides[1];
|
||||
final un = ((nv12
|
||||
? planes[1][uy * uvStride + ux * 2].toDouble()
|
||||
: planes[1][uy * uvStride + ux].toDouble()) -
|
||||
128.0) *
|
||||
(255.0 / 224.0);
|
||||
final vn = ((nv12
|
||||
? planes[1][uy * uvStride + ux * 2 + 1].toDouble()
|
||||
: planes[2][uy * uvStride + ux].toDouble()) -
|
||||
128.0) *
|
||||
(255.0 / 224.0);
|
||||
final r = yy + 1.402 * vn;
|
||||
final g = yy - 0.344136 * un - 0.714136 * vn;
|
||||
final b = yy + 1.772 * un;
|
||||
return (r, g, b);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user