red-future/observer
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
admin
2026-08-25 16:01:07 +08:00
parent 63b169a033
commit d3ba1c7e48
26 changed files with 1542 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flutter_app/android/app/build.gradle.kts
+4
View File
@@ -40,6 +40,10 @@ kotlin {
}
}
// 相机用 Android 框架 camera2 API 完全自研（CameraChannel.kt）：
// 预览 SurfaceTexture（Flutter 纹理）+ ImageReader 分析帧（原生侧旋转成竖屏后回传），
// 不依赖任何相机三方库（含 CameraX）。
// tflite_flutter 依赖的 tensorflow-lite / tensorflow-lite-gpu / tensorflow-lite-api 三个 AAR
// 声明了相同 namespace(org.tensorflow.lite),新 AGP 视作冲突直接报错;
// 本项目仅用 CPU 推理,GPU delegate 未使用,排除 gpu 及其传递依赖的 api 即可。
flutter_app/android/app/proguard-rules.pro
Vendored
+13
View File
@@ -0,0 +1,13 @@
# R8 release 压缩：okhttp 可选 TLS 平台类（BouncyCastle/Conscrypt/OpenJSSE）
# 与 tflite_flutter 的反射注解类未打包，仅需忽略引用告警（AGP missing_rules.txt 生成）
-dontwarn org.bouncycastle.jsse.BCSSLParameters
-dontwarn org.bouncycastle.jsse.BCSSLSocket
-dontwarn org.bouncycastle.jsse.provider.BouncyCastleJsseProvider
-dontwarn org.conscrypt.Conscrypt$Version
-dontwarn org.conscrypt.Conscrypt
-dontwarn org.conscrypt.ConscryptHostnameVerifier
-dontwarn org.openjsse.javax.net.ssl.SSLParameters
-dontwarn org.openjsse.javax.net.ssl.SSLSocket
-dontwarn org.openjsse.net.ssl.OpenJSSE
-dontwarn org.tensorflow.lite.InterpreterFactoryApi
-dontwarn org.tensorflow.lite.annotations.UsedByReflection
flutter_app/android/app/src/main/kotlin/com/example/observer/CameraChannel.kt
+568
View File
@@ -0,0 +1,568 @@
package com.example.observer
import android.graphics.Rect
import android.graphics.RectF
import android.graphics.SurfaceTexture
import android.os.Build
import android.hardware.camera2.CameraCaptureSession
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraDevice
import android.hardware.camera2.CameraManager
import android.hardware.camera2.CaptureRequest
import android.graphics.ImageFormat
import android.media.Image
import android.media.ImageReader
import android.os.Handler
import android.os.HandlerThread
import android.os.SystemClock
import android.util.Log
import android.util.Size
import android.view.Surface
import io.flutter.embedding.android.FlutterActivity
import io.flutter.embedding.engine.FlutterEngine
import io.flutter.plugin.common.EventChannel
import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel
import io.flutter.view.TextureRegistry
/**
* 自写原生相机：Android 框架 camera2 API 完全自研，零相机三方依赖（含 CameraX）。
*
* 核心：分析帧在原生侧旋转成竖屏方向再回传，Flutter 侧恒用 rotation=0，
* 与 iOS（camera_avfoundation，帧本来就是竖屏）行为对齐，彻底消除
* "横屏传感器帧 → 90° 旋转 + FIT_COVER crop 映射"的标注偏移根因。
*
* 旋转角来自设备标准值 sensorOrientation - displayRotation（camera2 特性，
* 非厂商 hack），因此映射跨厂商一致：任何设备上"分析帧 = 预览帧同 sensor
* 同旋转"，两者几何必然一致。
*
* 预览：SurfaceTexture 注册进 FlutterTextureRegistry，Flutter 侧 Texture widget
* 渲染（与插件 CameraPreview 相同的合成方式，overlay/诊断行/按钮可叠加；
* AndroidView+SurfaceView 会盖住 Flutter UI，不可用）。
*
* 通道：
* - MethodChannel "observer/camera"：start/stop/getZoomRange/setZoom/isStreaming/errorDescription
* - EventChannel "observer/camera/frames"：每帧 [width, height, rotationDegrees, bgra, bytes]
* - rotationDegrees 恒 0（帧已竖屏）；bgra=false（RGBA 字节序）
*/
class CameraChannel(
private val activity: FlutterActivity,
private val engine: FlutterEngine,
) {
private val messenger = engine.dartExecutor.binaryMessenger
private val method = MethodChannel(messenger, "observer/camera")
private val frames = EventChannel(messenger, "observer/camera/frames")
private val cameraManager: CameraManager = activity.getSystemService(CameraManager::class.java)
private var cameraDevice: CameraDevice? = null
private var captureSession: CameraCaptureSession? = null
private var imageReader: ImageReader? = null
private var surfaceEntry: TextureRegistry.SurfaceTextureEntry? = null
private var previewSurface: Surface? = null
private var eventSink: EventChannel.EventSink? = null
private var sensorOrientation = 90
private var activeArray = Rect(0, 0, 1920, 1080)
private var maxZoom = 1.0f
/// 预览/分析共用分辨率（传感器方向，从设备流配置动态选择）
private var previewSize = Size(1920, 1080)
@Volatile
private var running = false
@Volatile
private var errorDescription: String? = null
private val mainHandler = Handler(activity.mainLooper)
private val analysisThread = HandlerThread("observer-analysis").also { it.start() }
private val analysisHandler = Handler(analysisThread.looper)
@Volatile
private var lastEmitMs = 0L
// 诊断计数（frameListener 线程写，stats 轮询读）：
// 帧回调到达次数 / 成功发出 / 发出异常 / 无订阅者丢弃
@Volatile
private var frameCallbacks = 0L
@Volatile
private var emitOk = 0L
@Volatile
private var emitErr = 0L
@Volatile
private var sinkNullCount = 0L
// 实际发出帧尺寸（缩小后）
@Volatile
private var emitW = 0
@Volatile
private var emitH = 0
fun register() {
method.setMethodCallHandler(::onMethodCall)
frames.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
eventSink = events
}
override fun onCancel(arguments: Any?) {
eventSink = null
}
})
}
private fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
when (call.method) {
"start" -> start(result)
"stop" -> {
stop()
result.success(true)
}
"getZoomRange" -> result.success(listOf(1.0, maxZoom.toDouble()))
"setZoom" -> {
val v = (call.arguments as Number).toFloat()
setZoom(v)
result.success(true)
}
"isStreaming" -> result.success(running)
"errorDescription" -> result.success(errorDescription)
"stats" -> result.success(
mapOf(
"callbacks" to frameCallbacks,
"emitOk" to emitOk,
"emitErr" to emitErr,
"sinkNull" to sinkNullCount,
"sink" to (eventSink != null),
"running" to running,
"rotation" to rotation,
"quarterTurns" to displayDegrees / 90,
"displayDegrees" to displayDegrees,
"size" to "${previewSize.width}x${previewSize.height}",
"emitSize" to "${emitW}x$emitH",
"emitAgeMs" to
if (lastEmitMs == 0L) -1L
else SystemClock.elapsedRealtime() - lastEmitMs,
"error" to errorDescription,
),
)
else -> result.notImplemented()
}
}
private fun start(result: MethodChannel.Result) {
if (running) {
result.success(true)
return
}
running = true
errorDescription = null
try {
val cameraId = pickBackCamera() ?: throw IllegalStateException("未找到后置摄像头")
val characteristics = cameraManager.getCameraCharacteristics(cameraId)
sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 90
activeArray = characteristics.get(
CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE,
) ?: Rect(0, 0, 1920, 1080)
maxZoom = characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_MAX_DIGITAL_ZOOM) ?: 1.0f
// 分辨率不写死：从设备流配置查询（传感器方向尺寸，宽≥高），
// 预览与分析共用同一尺寸，保证两者几何一致（对齐根因）。
// 查询是优化而非必需：个别设备 getOutputSizes 会返回 null 或抛异常
// （实测某设备内部抛 getClass NPE），失败一律回退默认分辨率，
// 绝不让相机启动失败
try {
val streamMap = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
if (streamMap != null) {
val previewSizes =
streamMap.getOutputSizes(SurfaceTexture::class.java) ?: emptyArray()
val analysisSizes =
streamMap.getOutputSizes(ImageFormat.YUV_420_888) ?: emptyArray()
previewSize = pickSize(previewSizes, analysisSizes)
}
} catch (e: Exception) {
Log.w(TAG, "size query failed, fallback ${previewSize.width}x${previewSize.height}", e)
}
val entry = surfaceEntry ?: engine.getRenderer().createSurfaceTexture().also {
surfaceEntry = it
}
// 关键：SurfaceTexture 未设默认 buffer 尺寸时 createCaptureSession 会配置失败
// （camera2 按该尺寸做 stream 校验）。显式指定与 ImageReader 一致的分辨率，
// 保证预览与分析帧同分辨率同裁剪
entry.surfaceTexture().setDefaultBufferSize(previewSize.width, previewSize.height)
previewSurface?.release()
previewSurface = Surface(entry.surfaceTexture())
imageReader?.close()
// YUV_420_888 是 camera2 对所有设备保证支持的 ImageReader 输出格式；
// RGBA_8888 个别设备不支持（实测：查询 NPE + 配置失败双连败），弃用
imageReader = ImageReader.newInstance(
previewSize.width, previewSize.height, ImageFormat.YUV_420_888, 2,
).also { it.setOnImageAvailableListener(frameListener, analysisHandler) }
cameraManager.openCamera(
cameraId,
object : CameraDevice.StateCallback() {
override fun onOpened(device: CameraDevice) {
cameraDevice = device
createSession(result)
}
override fun onDisconnected(device: CameraDevice) {
device.close()
cameraDevice = null
}
override fun onError(device: CameraDevice, error: Int) {
device.close()
cameraDevice = null
running = false
errorDescription = "camera open error $error"
result.error("start", errorDescription, null)
}
},
mainHandler,
)
} catch (e: Exception) {
running = false
errorDescription = e.toString()
// details 带完整堆栈：任何残留异常都能在诊断行看到精确位置
result.error("start", e.toString(), Log.getStackTraceString(e))
}
}
private fun createSession(result: MethodChannel.Result) {
val device = cameraDevice ?: return
val surfaces = listOfNotNull(previewSurface, imageReader?.surface)
device.createCaptureSession(
surfaces,
object : CameraCaptureSession.StateCallback() {
override fun onConfigured(session: CameraCaptureSession) {
captureSession = session
try {
val request = buildRequest(session.device, surfaces)
session.setRepeatingRequest(request, null, mainHandler)
result.success(
mapOf(
"textureId" to (surfaceEntry?.id() ?: -1L),
// 传感器方向尺寸（宽≥高）：Flutter 侧 SizedBox 在 RotatedBox
// 内部声明纹理尺寸，旋转后视觉上才是竖屏 1080x1920
"w" to previewSize.width,
"h" to previewSize.height,
// 预览旋转 = 显示旋转（不是 rotation/90！）：
// Flutter 引擎渲染 Texture 时已自动应用 SurfaceTexture
// 变换矩阵（传感器方向补偿），预览只需再按显示旋转补偿
"quarterTurns" to displayDegrees / 90,
// 诊断：旋转计算输入值
"sensorOrientation" to sensorOrientation,
"displayDegrees" to displayDegrees,
),
)
} catch (e: Exception) {
errorDescription = "onConfigured: $e"
result.error("start", errorDescription, Log.getStackTraceString(e))
}
}
override fun onConfigureFailed(session: CameraCaptureSession) {
running = false
errorDescription = "capture session configure failed"
result.error("start", errorDescription, null)
}
},
mainHandler,
)
}
private fun buildRequest(device: CameraDevice, targets: List<Surface>): CaptureRequest {
val builder = device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
targets.forEach { builder.addTarget(it) }
builder.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
// 连续对焦：无 AF 能力的设备忽略该设置
try {
builder.set(
CaptureRequest.CONTROL_AF_MODE,
CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE,
)
} catch (_: IllegalArgumentException) {
}
return builder.build()
}
private fun setZoom(z: Float) {
val session = captureSession ?: return
val clamped = z.coerceIn(1.0f, maxZoom)
val r = activeArray
val insetW = r.width() * (1 - 1 / clamped) / 2
val insetH = r.height() * (1 - 1 / clamped) / 2
val crop = RectF(
r.left + insetW,
r.top + insetH,
r.right - insetW,
r.bottom - insetH,
)
try {
val surface = previewSurface ?: return
val builder = session.device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
builder.addTarget(surface)
imageReader?.surface?.let { builder.addTarget(it) }
builder.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
builder.set(
CaptureRequest.SCALER_CROP_REGION,
Rect(
crop.left.toInt(),
crop.top.toInt(),
crop.right.toInt(),
crop.bottom.toInt(),
),
)
session.setRepeatingRequest(builder.build(), null, mainHandler)
} catch (e: Exception) {
Log.w(TAG, "setZoom failed", e)
}
}
private fun stop() {
running = false
try {
captureSession?.close()
} catch (_: Exception) {
}
captureSession = null
try {
cameraDevice?.close()
} catch (_: Exception) {
}
cameraDevice = null
}
fun destroy() {
stop()
imageReader?.close()
imageReader = null
previewSurface?.release()
previewSurface = null
surfaceEntry?.release()
surfaceEntry = null
analysisThread.quitSafely()
}
private fun pickBackCamera(): String? {
val ids = cameraManager.cameraIdList
for (id in ids) {
val c = cameraManager.getCameraCharacteristics(id)
val facing = c.get(CameraCharacteristics.LENS_FACING)
if (facing == CameraCharacteristics.LENS_FACING_BACK) return id
}
return ids.firstOrNull()
}
/**
* 从设备流配置挑预览/分析共用分辨率（均为传感器方向尺寸，宽≥高）：
* 取两者交集，16:9 优先、长边 ≤1920 内取最大（推理输入 640x640，
* 更高只增帧传输与旋转开销，无精度收益）；无 16:9 时退回最大交集尺寸。
*/
private fun pickSize(previewSizes: Array<Size>, analysisSizes: Array<Size>): Size {
val common = previewSizes.filter { analysisSizes.contains(it) }
val fallback = common.maxByOrNull { it.width.toLong() * it.height }
?: return Size(1920, 1080)
return common
.filter {
Math.abs(it.width.toDouble() / it.height - 16.0 / 9.0) < 0.03 &&
it.width <= 1920 && it.height <= 1920
}
.maxByOrNull { it.width.toLong() * it.height }
?: fallback
}
/**
* 显示旋转角（度）。Display.ROTATION_* 在 API 36 是受限常量，直接用其值（0/1/2/3）。
* API 30+ 用 activity.display（现代 API，实测可靠跟踪旋转）；
* 旧版 windowManager.defaultDisplay.rotation 已废弃，实测在部分设备恒返回 0。
*/
private val displayDegrees: Int
get() = try {
val rot: Int = if (Build.VERSION.SDK_INT >= 30) {
activity.display?.rotation ?: 0
} else {
@Suppress("DEPRECATION")
activity.windowManager.defaultDisplay.rotation
}
when (rot) {
1 -> 90
2 -> 180
3 -> 270
else -> 0
}
} catch (e: Exception) {
Log.w(TAG, "display rotation query failed", e)
0
}
/** 传感器 → 竖屏显示的顺时针旋转角（后摄无镜像） */
private val rotation: Int
get() = ((sensorOrientation - displayDegrees) % 360 + 360) % 360
private val frameListener = ImageReader.OnImageAvailableListener { reader ->
// 诊断计数：回调是否到达（放在一切判定之前，任何丢弃都先计数）
frameCallbacks++
if (frameCallbacks % 50 == 0L) {
Log.i(TAG, "frames cb=$frameCallbacks ok=$emitOk err=$emitErr sinkNull=$sinkNullCount")
}
val image = reader.acquireLatestImage() ?: return@OnImageAvailableListener
try {
val sink = eventSink
if (sink == null) {
// 诊断：有帧但无订阅者（Dart 侧订阅未建立/被取消）
sinkNullCount++
errorDescription = "帧监听无订阅者"
return@OnImageAvailableListener
}
if (!running) return@OnImageAvailableListener
val now = SystemClock.elapsedRealtime()
// 节流：推理 ~100ms 一帧，避免数 MB 帧无谓传输
if (now - lastEmitMs < 100) return@OnImageAvailableListener
lastEmitMs = now
val bytes = yuvToRgba(image, rotation)
// 缩小后再发：8.3MB/帧对主线程编码与通道传输都过重，540x960 足够
// （推理输入 704，坐标归一化，映射不受分辨率影响）
val scaled = downscaleToFit(
bytes[0] as Int, bytes[1] as Int, bytes[2] as ByteArray, 960,
)
emitW = scaled[0] as Int
emitH = scaled[1] as Int
if (emitOk == 0L) Log.i(TAG, "first emit ${emitW}x${emitH}")
// 关键：EventSink.success 内部 FlutterJNI.dispatchPlatformMessage 强制主线程
// （ensureRunningOnMainThread 抛 RuntimeException），后台线程直接调必失败——
// 曾因此每帧抛"必须主线程"异常、Dart 侧永远 流:0。必须 post 到主线程发送
mainHandler.post {
try {
sink.success(listOf(scaled[0], scaled[1], 0, false, scaled[2]))
emitOk++
} catch (e: Exception) {
emitErr++
Log.e(TAG, "emit frame failed", e)
errorDescription = "帧异常: ${e.javaClass.simpleName}: ${e.message}"
}
}
} catch (e: Exception) {
emitErr++
Log.e(TAG, "emit frame failed", e)
errorDescription = "帧异常: ${e.javaClass.simpleName}: ${e.message}"
} finally {
image.close()
}
}
/**
* YUV_420_888 三平面 → RGBA 单平面，同时顺时针旋转成竖屏。
* 返回 [width, height, bytes]。整数 BT.601 转换；U/V 兼容 planar
* （pixelStride=1）与 semi-planar（pixelStride=2，NV21 式交错）布局。
* dst(x', y') = src(y, W-1-x')（90° 顺时针），逐像素处理 rowStride 填充。
*/
private fun yuvToRgba(image: Image, deg: Int): List<Any> {
val yPlane = image.planes[0]
val uPlane = image.planes[1]
val vPlane = image.planes[2]
val srcW = image.width
val srcH = image.height
val yStride = yPlane.rowStride
val uStride = uPlane.rowStride
val vStride = vPlane.rowStride
val uPixel = uPlane.pixelStride
val vPixel = vPlane.pixelStride
val yBuf = yPlane.buffer
val uBuf = uPlane.buffer
val vBuf = vPlane.buffer
val dstW = if (deg == 90 || deg == 270) srcH else srcW
val dstH = if (deg == 90 || deg == 270) srcW else srcH
val dst = ByteArray(dstW * dstH * 4)
for (dy in 0 until dstH) {
val rowBase = dy * dstW * 4
for (dx in 0 until dstW) {
val sx: Int
val sy: Int
when (deg) {
90 -> {
sx = dy
sy = srcH - 1 - dx
}
180 -> {
sx = srcW - 1 - dx
sy = srcH - 1 - dy
}
270 -> {
sx = srcW - 1 - dy
sy = dx
}
else -> {
sx = dx
sy = dy
}
}
val yy = (yBuf.get(sy * yStride + sx).toInt() and 0xFF) - 16
val uu = (uBuf.get((sy / 2) * uStride + (sx / 2) * uPixel).toInt() and 0xFF) - 128
val vv = (vBuf.get((sy / 2) * vStride + (sx / 2) * vPixel).toInt() and 0xFF) - 128
val r = ((298 * yy + 409 * vv + 128) shr 8).coerceIn(0, 255)
val g = ((298 * yy - 100 * uu - 208 * vv + 128) shr 8).coerceIn(0, 255)
val b = ((298 * yy + 516 * uu + 128) shr 8).coerceIn(0, 255)
val di = rowBase + dx * 4
dst[di] = r.toByte()
dst[di + 1] = g.toByte()
dst[di + 2] = b.toByte()
dst[di + 3] = 0xFF.toByte()
}
}
return listOf(dstW, dstH, dst)
}
/**
* RGBA 帧整数倍缩小，长边 ≤ maxLong 时原样返回（f=1）。
* 2x2 盒式平均（对 640x640 推理输入足够；比最近邻平滑，颜色更准）。
*/
private fun downscaleToFit(w: Int, h: Int, rgba: ByteArray, maxLong: Int): List<Any> {
val long = maxOf(w, h)
if (long <= maxLong) return listOf(w, h, rgba)
val f = (long + maxLong - 1) / maxLong
val dw = w / f
val dh = h / f
val out = ByteArray(dw * dh * 4)
for (dy in 0 until dh) {
val y0 = dy * f
val y1 = minOf(y0 + f, h)
val rows = y1 - y0
for (dx in 0 until dw) {
val x0 = dx * f
val x1 = minOf(x0 + f, w)
var r = 0L
var g = 0L
var b = 0L
var a = 0L
for (sy in y0 until y1) {
var si = sy * w * 4 + x0 * 4
for (sx in x0 until x1) {
r += rgba[si].toInt() and 0xFF
g += rgba[si + 1].toInt() and 0xFF
b += rgba[si + 2].toInt() and 0xFF
a += rgba[si + 3].toInt() and 0xFF
si += 4
}
}
val n = rows * (x1 - x0)
val di = (dy * dw + dx) * 4
out[di] = (r / n).toByte()
out[di + 1] = (g / n).toByte()
out[di + 2] = (b / n).toByte()
out[di + 3] = (a / n).toByte()
}
}
return listOf(dw, dh, out)
}
companion object {
private const val TAG = "CameraChannel"
}
}
flutter_app/android/app/src/main/kotlin/com/example/observer/MainActivity.kt
+18 -1
View File
@@ -1,5 +1,22 @@
package com.example.observer
import io.flutter.embedding.android.FlutterActivity
import io.flutter.embedding.engine.FlutterEngine
class MainActivity : FlutterActivity()
class MainActivity : FlutterActivity() {
private var cameraChannel: CameraChannel? = null
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
// 自写原生相机（替代 camera_android_camerax 插件）：
// 分析帧在原生侧旋转成竖屏后回传，Flutter 侧恒 rotation=0（对齐 iOS）。
// configureFlutterEngine（onCreate 阶段）只注册通道与 viewFactory，
// 实际 bindToLifecycle 由 Flutter 相机页 start 时触发（此时已 RESUMED）
cameraChannel = CameraChannel(this, flutterEngine).also { it.register() }
}
override fun onDestroy() {
cameraChannel?.destroy()
super.onDestroy()
}
}
flutter_app/lib/camera/app_camera_controller.dart
+243 -34
View File
@@ -1,61 +1,257 @@
import 'dart:async';
import 'package:camera/camera.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'frame_analyzer.dart';
/// camera 插件封装：后摄图像流（对应 Kotlin CameraController）。
class AppCameraController {
/// 相机抽象（对应 Kotlin CameraController）：
///
/// - Android：自写原生通道（[NativeCameraController]）。分析帧在 Kotlin 侧
/// 旋转成竖屏后经 EventChannel 回传，Flutter 侧恒 rotation=0——与 iOS
/// （camera_avfoundation 插件，帧本来就是竖屏方向）行为一致，消除
/// "横屏传感器帧 → 90° 旋转 + FIT_COVER crop 映射"的标注偏移根因。
/// - iOS：camera 插件（原逻辑）。帧已竖屏，rotation 恒 0。
abstract class AppCameraController {
static Future<AppCameraController?> create() async {
if (defaultTargetPlatform == TargetPlatform.android) {
return NativeCameraController();
}
return PluginCameraController.create();
}
/// 分析流回调实际触发次数（诊断用，与 analyzer 帧计数区分）
int get streamCallbacks;
bool get isInitialized;
bool get isStreaming;
String? get errorDescription;
/// 检测框 overlay 应使用的旋转角。两平台帧都已竖屏 → 恒 0。
int get rotationDegrees => 0;
/// 诊断：传感器方向 / 显示旋转（仅 Android 原生通道上报）
int? get sensorOrientation => null;
int? get displayDegrees => null;
/// 预览实际应用的旋转圈数（仅 Android 原生通道上报，诊断用）
int get quarterTurns => -1;
/// 诊断：轮询原生侧帧状态（Android 返回计数；iOS 返回空）
Future<Map<dynamic, dynamic>> stats() async => const {};
Future<void> start(FrameAnalyzer analyzer);
Future<void> stop();
Future<double> getMinZoomLevel();
Future<double> getMaxZoomLevel();
Future<void> setZoomLevel(double value);
/// 预览 widget：Android 为原生 SurfaceView（AndroidView），iOS 为插件纹理
Widget buildPreview();
}
/// Android：自写原生相机通道（Kotlin CameraChannel）。
class NativeCameraController extends AppCameraController {
static const MethodChannel _channel = MethodChannel('observer/camera');
static const EventChannel _frames = EventChannel('observer/camera/frames');
StreamSubscription<dynamic>? _sub;
bool _streaming = false;
String? _error;
/// 原生侧注册的预览纹理（SurfaceTexture）
int? _textureId;
int _textureW = 1920;
int _textureH = 1080;
int _quarterTurns = 1;
@override
int streamCallbacks = 0;
int? _sensorOrientation;
int? _displayDegrees;
@override
int? get sensorOrientation => _sensorOrientation;
@override
int? get displayDegrees => _displayDegrees;
@override
int get quarterTurns => _quarterTurns;
@override
bool get isInitialized => _streaming;
@override
bool get isStreaming => _streaming;
@override
String? get errorDescription => _error;
@override
Future<Map<dynamic, dynamic>> stats() async {
try {
final r = await _channel.invokeMethod<Map<dynamic, dynamic>>('stats');
if (r != null) {
// 旋转/显示角度随轮询实时刷新：手机旋转后预览与帧旋转都跟着变
final dd = r['displayDegrees'];
if (dd is int) _displayDegrees = dd;
final turns = r['quarterTurns'];
if (turns is int) _quarterTurns = turns;
}
return r ?? const {};
} catch (e) {
return {'pollErr': '$e'};
}
}
@override
Future<void> start(FrameAnalyzer analyzer) async {
await stop();
// 先订阅再启动：原生 start 绑定后立刻推帧，避免首帧竞态
_sub = _frames.receiveBroadcastStream().listen((event) {
// 计数放最前：任何到达的事件都先记账，后续解析失败也不丢计数
streamCallbacks++;
try {
final f = event as List;
final w = f[0] as int;
final h = f[1] as int;
final rotation = f[2] as int;
final bgra = f[3] as bool;
final bytes = f[4] as Uint8List;
analyzer.analyzeRaw(
planes: [bytes],
strides: [w * 4],
width: w,
height: h,
isBgra: true,
rgbaOrder: !bgra,
rotationDegrees: rotation,
);
} catch (e) {
_error = '帧解析: $e';
analyzer.recordStreamError('frames parse: $e');
}
}, onError: (Object e) {
_error = '$e';
analyzer.recordStreamError('frames: $e');
});
analyzer.reset();
analyzer.worker?.reset();
try {
final r = await _channel.invokeMethod<Map<dynamic, dynamic>>('start');
_textureId = r?['textureId'] as int?;
_textureW = (r?['w'] as num?)?.toInt() ?? _textureW;
_textureH = (r?['h'] as num?)?.toInt() ?? _textureH;
_quarterTurns = (r?['quarterTurns'] as num?)?.toInt() ?? 1;
_sensorOrientation = (r?['sensorOrientation'] as num?)?.toInt();
_displayDegrees = (r?['displayDegrees'] as num?)?.toInt();
} catch (e) {
_error = '$e';
analyzer.recordStreamError('camera start: $e');
rethrow;
}
_streaming = true;
}
@override
Future<void> stop() async {
_streaming = false;
await _sub?.cancel();
_sub = null;
try {
await _channel.invokeMethod('stop');
} catch (_) {}
}
@override
Future<double> getMinZoomLevel() async {
final r = await _channel.invokeMethod<List<dynamic>>('getZoomRange');
return (r != null && r.isNotEmpty ? (r[0] as num).toDouble() : 1.0);
}
@override
Future<double> getMaxZoomLevel() async {
final r = await _channel.invokeMethod<List<dynamic>>('getZoomRange');
return (r != null && r.length > 1 ? (r[1] as num).toDouble() : 1.0);
}
@override
Future<void> setZoomLevel(double value) async {
try {
await _channel.invokeMethod('setZoom', value);
} catch (_) {}
}
/// 预览纹理：Flutter 引擎渲染 Texture 时已自动应用 SurfaceTexture 变换矩阵
/// （传感器方向补偿，内容已转成自然方向的竖屏），因此：
/// 1. 区域声明旋转后的尺寸（宽高互换）——否则横屏区域会把竖屏内容横向拉伸
/// 2. RotatedBox 只按显示旋转补偿（quarterTurns = 屏转/90，竖屏 0 / 横屏 1）
/// 再 FittedBox cover（= CoordinateMapper 的 FIT_COVER 数学一致）填满全屏。
/// 不用 AndroidView+SurfaceView——平台视图会盖住 Flutter UI（诊断行/设置按钮/overlay）
@override
Widget buildPreview() {
final id = _textureId;
if (id == null) return const SizedBox.shrink();
return FittedBox(
fit: BoxFit.cover,
child: RotatedBox(
quarterTurns: _quarterTurns % 4,
child: SizedBox(
width: _textureH.toDouble(),
height: _textureW.toDouble(),
child: Texture(textureId: id),
),
),
);
}
}
/// iOS：camera 插件封装（原实现）。
class PluginCameraController extends AppCameraController {
final List<CameraDescription> cameras;
CameraController? controller;
/// 图像流回调实际触发次数（诊断用，与 analyzer 帧计数区分）
@override
int streamCallbacks = 0;
AppCameraController._(this.cameras);
PluginCameraController._(this.cameras);
static Future<AppCameraController?> create() async {
static Future<PluginCameraController?> create() async {
final cameras = await availableCameras();
if (cameras.isEmpty) return null;
return AppCameraController._(cameras);
return PluginCameraController._(cameras);
}
@override
bool get isInitialized => controller?.value.isInitialized ?? false;
CameraController get currentController =>
controller ?? (throw StateError('camera not initialized'));
@override
bool get isStreaming => controller?.value.isStreamingImages ?? false;
/// 图像流送达时的旋转角（传感器 → 竖屏显示所需的顺时针旋转）。
/// 与 CameraX rotationDegrees 同公式；预览本身由平台旋转，检测框 overlay
/// 用同一角度映射即可对齐。
int get rotationDegrees {
final c = controller;
if (c == null) return 0;
final deviceDegrees = switch (c.value.deviceOrientation) {
DeviceOrientation.portraitUp => 0,
DeviceOrientation.landscapeLeft => 90,
DeviceOrientation.portraitDown => 180,
DeviceOrientation.landscapeRight => 270,
};
final sensor = c.description.sensorOrientation;
final isFront =
c.description.lensDirection == CameraLensDirection.front;
final degrees = (isFront ? sensor + deviceDegrees : sensor - deviceDegrees) % 360;
return degrees < 0 ? degrees + 360 : degrees;
}
@override
String? get errorDescription => controller?.value.errorDescription;
@override
Future<void> start(FrameAnalyzer analyzer) async {
await stop();
final desc = cameras.firstWhere(
(c) => c.lensDirection == CameraLensDirection.back,
orElse: () => cameras.first);
// iOS 用默认 bgra8888（420v 在部分 iOS 版本上视频输出静默不送帧），
// Android 用 yuv420 多平面。
final fmt = defaultTargetPlatform == TargetPlatform.iOS
? ImageFormatGroup.bgra8888
: ImageFormatGroup.yuv420;
final c = CameraController(desc, ResolutionPreset.high,
enableAudio: false, imageFormatGroup: fmt);
// iOS image stream 帧已按竖屏方向输出（无需旋转），
// 与 Android 原生通道（帧原生侧旋转成竖屏）统一 rotation=0
final c = CameraController(desc, ResolutionPreset.veryHigh,
enableAudio: false, imageFormatGroup: ImageFormatGroup.bgra8888);
controller = c;
await c.initialize();
// 相机（重新）启动后重置运动/背景参考与抽帧节流，避免旧场景残留
@@ -66,7 +262,7 @@ class AppCameraController {
await c.startImageStream((image) {
streamCallbacks++;
try {
analyzer.analyze(image, rotationDegrees);
analyzer.analyze(image, 0);
} catch (e, st) {
debugPrint('[camera] analyze error: $e\n$st');
analyzer.recordStreamError('analyze: $e');
@@ -80,6 +276,7 @@ class AppCameraController {
}
}
@override
Future<void> stop() async {
final c = controller;
if (c == null) return;
@@ -89,4 +286,16 @@ class AppCameraController {
} catch (_) {}
await c.dispose();
}
@override
Future<double> getMinZoomLevel() => controller!.getMinZoomLevel();
@override
Future<double> getMaxZoomLevel() => controller!.getMaxZoomLevel();
@override
Future<void> setZoomLevel(double value) => controller!.setZoomLevel(value);
@override
Widget buildPreview() => CameraPreview(controller!);
}
flutter_app/lib/camera/camera_screen.dart
+121 -40
View File
@@ -1,7 +1,6 @@
import 'dart:ui' show PlatformDispatcher;
import 'dart:async';
import 'dart:ui' as ui show PlatformDispatcher;
import 'package:camera/camera.dart';
import 'package:flutter/foundation.dart' show defaultTargetPlatform;
import 'package:flutter/material.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:wakelock_plus/wakelock_plus.dart';
@@ -30,17 +29,88 @@ class _CameraScreenState extends State<CameraScreen> {
String? _globalError;
String? _initError;
/// 置信度阈值（设置页滑块调整，worker 内实时生效）
double _minScore = 0.10;
/// 原生侧帧状态轮询结果（诊断用；无帧时诊断行也能实时刷新）
Map<dynamic, dynamic> _nativeStats = const {};
Timer? _statsTimer;
void _openSettings() {
final vm = _viewModel;
if (vm == null) return;
showModalBottomSheet<void>(
context: context,
backgroundColor: Colors.black87,
builder: (ctx) => StatefulBuilder(
builder: (ctx, setSheetState) => Padding(
padding: const EdgeInsets.all(20),
child: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
const Text('识别设置',
style: TextStyle(
color: Colors.white, fontSize: 16, fontWeight: FontWeight.bold)),
const SizedBox(height: 12),
Row(
children: [
const Text('置信度阈值',
style: TextStyle(color: Colors.white70, fontSize: 14)),
const Spacer(),
Text('${(_minScore * 100).toStringAsFixed(0)}%',
style: const TextStyle(
color: Colors.greenAccent,
fontSize: 14,
fontWeight: FontWeight.bold)),
],
),
Slider(
value: _minScore,
min: 0.05,
max: 0.50,
divisions: 45,
activeColor: Colors.greenAccent,
onChanged: (v) {
setSheetState(() => _minScore = v);
_analyzer?.worker?.setMinScore(v);
},
),
const SizedBox(height: 8),
const Text(
'阈值越低识别越灵敏（低分框越多，误报也可能增加）；'
'野鸡模型置信度普遍在 10%~20%，场景识别不到时可适当调低。',
style: TextStyle(color: Colors.white54, fontSize: 12),
),
],
),
),
),
);
}
@override
void initState() {
super.initState();
final oldPlatform = PlatformDispatcher.instance.onError;
PlatformDispatcher.instance.onError = (error, stack) {
setState(() => _globalError = 'Platform: $error');
final oldPlatform = ui.PlatformDispatcher.instance.onError;
ui.PlatformDispatcher.instance.onError = (error, stack) {
setState(() => _globalError =
'Platform: $error\n${stack.toString().split('\n').take(3).join('\n')}');
return oldPlatform?.call(error, stack) ?? false;
};
WidgetsBinding.instance.addPostFrameCallback((_) => _init());
// 相机页常亮：野外观察时保持屏幕不熄（离开页面时关闭）
WakelockPlus.enable();
// 每秒轮询原生侧帧状态：无帧时诊断行也能实时刷新（camErr/计数）
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _pollStats());
}
Future<void> _pollStats() async {
final camera = _cameraController;
if (camera == null) return;
final s = await camera.stats();
if (!mounted) return;
setState(() => _nativeStats = s);
}
Future<void> _init() async {
@@ -100,6 +170,7 @@ class _CameraScreenState extends State<CameraScreen> {
@override
void dispose() {
_statsTimer?.cancel();
WakelockPlus.disable();
_cameraController?.stop();
_analyzer?.dispose();
@@ -120,29 +191,24 @@ class _CameraScreenState extends State<CameraScreen> {
if (!_permissionGranted)
_PermissionGuide(onRequest: () => _init())
else if (vm != null && (camera?.isInitialized ?? false))
// 预览 + 检测框同几何：overlay 作为 CameraPreview 的 child，
// 与纹理共享同一 Stack/尺寸，避免比例或裁剪导致的位置偏移
// 预览 + 检测框同几何：overlay 作为预览 widget 的 sibling，
// 与纹理共享同一 Stack/尺寸，避免比例或裁剪导致的位置偏移。
// 两平台分析帧都已竖屏（Android 原生侧旋转 / iOS 插件本来就竖屏），
// rotation 恒 0，CoordinateMapper 走纯 FIT_COVER 缩放路径
ListenableBuilder(
listenable: vm,
builder: (context, _) => _ZoomablePreview(
controller: camera!.currentController,
imageWidthPx: vm.state.imageWidthPx,
imageHeightPx: vm.state.imageHeightPx,
controller: camera!,
overlay: DetectionOverlay(
results: vm.state.results,
// iOS 纹理不旋转显示（_wrapInRotatedBox 仅 Android），
// 显示方向 = buffer 原样 = 检测方向，旋转必须为 0；
// Android 纹理被 RotatedBox 旋转，需用插件报告的 rotation。
rotation: defaultTargetPlatform == TargetPlatform.iOS
? 0
: vm.state.rotation,
rotation: 0,
imageWidthPx: vm.state.imageWidthPx,
imageHeightPx: vm.state.imageHeightPx,
),
),
)
else if (camera?.isInitialized ?? false)
_ZoomablePreview(controller: camera!.currentController)
_ZoomablePreview(controller: camera!)
else
const Center(
child: Text('相机启动中…', style: TextStyle(color: Colors.white70)),
@@ -197,6 +263,7 @@ class _CameraScreenState extends State<CameraScreen> {
right: 0,
child: _CameraTopBar(
onClose: () => Navigator.of(context).pop(),
onOpenSettings: _openSettings,
),
),
],
@@ -238,13 +305,25 @@ class _CameraScreenState extends State<CameraScreen> {
crossAxisAlignment: CrossAxisAlignment.center,
children: [
Text(
'模型:${vm.state.modelReady ? '已加载' : '未加载'} 帧:${vm.state.framesReceived} 流:${camera?.streamCallbacks ?? 0} 推理:${vm.state.debugDetectCalls}次 异常:${vm.state.debugDetectErrors}次 处理:${vm.state.debugLastMs}ms 最高分:${(vm.state.debugHighestScore * 100).toStringAsFixed(1)}% 图:${vm.state.imageWidthPx}x${vm.state.imageHeightPx} 旋:${vm.state.rotation}',
'阈值:${(_minScore * 100).toStringAsFixed(0)}% 模型:${vm.state.modelReady ? '已加载' : '未加载'} 帧:${vm.state.framesReceived} 流:${camera?.streamCallbacks ?? 0} 推理:${vm.state.debugDetectCalls}次 异常:${vm.state.debugDetectErrors}次 处理:${vm.state.debugLastMs}ms 最高分:${(vm.state.debugHighestScore * 100).toStringAsFixed(1)}% 图:${vm.state.imageWidthPx}x${vm.state.imageHeightPx} 传感:${camera?.sensorOrientation ?? '-'} 屏转:${camera?.displayDegrees ?? '-'} 旋:${camera?.rotationDegrees ?? 0} turn:${camera?.quarterTurns ?? '-'}',
style: const TextStyle(color: Colors.white70, fontSize: 12),
),
if (_nativeStats.isNotEmpty)
Text(
'原生:回调${_nativeStats['callbacks'] ?? '-'} 发出${_nativeStats['emitOk'] ?? '-'} 异常${_nativeStats['emitErr'] ?? '-'} 无订阅${_nativeStats['sinkNull'] ?? '-'} sink:${_nativeStats['sink'] ?? '-'} 配置:${_nativeStats['size'] ?? '-'} 发帧:${_nativeStats['emitSize'] ?? '-'} 错误:${_nativeStats['error'] ?? '无'} 轮询:${_nativeStats['pollErr'] ?? 'ok'}${vm.state.results.isEmpty ? '' : ' 框1:(${vm.state.results.first.left.toStringAsFixed(2)},${vm.state.results.first.top.toStringAsFixed(2)},${vm.state.results.first.right.toStringAsFixed(2)},${vm.state.results.first.bottom.toStringAsFixed(2)})'}',
style: const TextStyle(
color: Colors.amberAccent, fontSize: 11),
),
if (vm.state.debugYuv.isNotEmpty)
Text(
'yuv:${vm.state.debugYuv}',
style: const TextStyle(
color: Colors.yellowAccent, fontSize: 11),
),
if (camera != null)
Text(
'streaming:${camera.currentController.value.isStreamingImages} '
'camErr:${camera.currentController.value.errorDescription ?? '无'}',
'streaming:${camera.isStreaming} '
'camErr:${camera.errorDescription ?? '无'}',
maxLines: 2,
overflow: TextOverflow.ellipsis,
style: const TextStyle(
@@ -274,24 +353,19 @@ class _CameraScreenState extends State<CameraScreen> {
],
);
}
}
/// 双指捏合缩放预览；overlay 与纹理同几何（CameraPreview child）
/// 双指捏合缩放预览；overlay 与纹理同几何（Stack 内同尺寸）
class _ZoomablePreview extends StatefulWidget {
final CameraController controller;
final AppCameraController controller;
/// 检测框 overlay（随帧更新，作为 CameraPreview 的 child 与纹理同区域）
/// 检测框 overlay（随帧更新，与纹理同区域）
final Widget? overlay;
/// 当前帧图像尺寸（用于按 buffer 比例约束预览，保证无拉伸变形）
final int imageWidthPx;
final int imageHeightPx;
const _ZoomablePreview({
required this.controller,
this.overlay,
this.imageWidthPx = 0,
this.imageHeightPx = 0,
});
@override
@@ -317,7 +391,9 @@ class _ZoomablePreviewState extends State<_ZoomablePreview> {
@override
Widget build(BuildContext context) {
final preview = GestureDetector(
// 预览铺满全屏（cover 裁剪由插件按视图比例完成）；
// overlay 与纹理同几何：作为 Stack sibling 叠在上层，坐标与纹理区域一致
return GestureDetector(
onScaleStart: (_) => _gestureStartZoom = _currentZoom,
onScaleUpdate: (d) {
final target =
@@ -326,24 +402,24 @@ class _ZoomablePreviewState extends State<_ZoomablePreview> {
_currentZoom = target;
widget.controller.setZoomLevel(target);
},
child: CameraPreview(widget.controller, child: widget.overlay),
);
final w = widget.imageWidthPx.toDouble();
final h = widget.imageHeightPx.toDouble();
if (w <= 0 || h <= 0) return preview;
// 按 buffer 比例约束显示区域：纹理与 overlay 同区域等比显示（无变形）
return Center(
child: AspectRatio(aspectRatio: w / h, child: preview),
child: Stack(
fit: StackFit.expand,
children: [
widget.controller.buildPreview(),
if (widget.overlay != null) widget.overlay!,
],
),
);
}
}
class _CameraTopBar extends StatelessWidget {
final VoidCallback onClose;
final VoidCallback onOpenSettings;
const _CameraTopBar({
required this.onClose,
required this.onOpenSettings,
});
@override
@@ -359,6 +435,11 @@ class _CameraTopBar extends StatelessWidget {
icon: const Icon(Icons.arrow_back, color: Colors.white),
onPressed: onClose,
),
IconButton(
tooltip: '识别设置',
icon: const Icon(Icons.tune, color: Colors.white70),
onPressed: onOpenSettings,
),
],
),
);
flutter_app/lib/camera/camera_view_model.dart
+5 -4
View File
@@ -4,7 +4,6 @@ import 'package:flutter/foundation.dart';
import '../detection/detection_result.dart';
import '../detection/motion_aggregator.dart';
import '../detection/tflite_detector.dart';
import '../reminder/reminder.dart';
@immutable
@@ -20,6 +19,7 @@ class CameraUiState {
final String? debugLastError;
final int framesReceived;
final int debugLastMs;
final String debugYuv;
const CameraUiState({
this.modelReady = false,
@@ -33,20 +33,19 @@ class CameraUiState {
this.debugLastError,
this.framesReceived = 0,
this.debugLastMs = 0,
this.debugYuv = '',
});
}
/// 检测结果置信度分级与轨迹确认。
///
/// - [lowConf]（模型阈值 0.10）：低于此分的框在检测阶段已丢弃。
/// - [highConf]（0.35）：高于此分直接确认显示；真实野鸡多为 0.1~0.2，
/// 高于 0.35 视为强证据。
/// - 0.10~0.35 之间：需要多帧稳定（[confirmFrames] 帧）或 活动证据
/// - 低于 0.35 的框：需要多帧稳定（[confirmFrames] 帧）或 活动证据
/// （运动区域/背景新出现区域重叠）才确认显示。
class CameraViewModel extends ChangeNotifier {
static const int maxTracks = 30;
static const double motionBoost = 0.15;
static const double lowConf = TfliteDetector.minScore;
static const double highConf = 0.35;
static const int confirmFrames = 3;
static const double associateRadius = 0.12;
@@ -86,6 +85,7 @@ class CameraViewModel extends ChangeNotifier {
String? lastError,
int framesReceived = 0,
int lastProcessMs = 0,
String yuvDiag = '',
}) {
final now = DateTime.now().millisecondsSinceEpoch;
_associate(results, motionRegions, noveltyRegions, now);
@@ -132,6 +132,7 @@ class CameraViewModel extends ChangeNotifier {
debugLastError: lastError,
framesReceived: framesReceived,
debugLastMs: lastProcessMs,
debugYuv: yuvDiag.isNotEmpty ? yuvDiag : _state.debugYuv,
);
notifyListeners();
}
flutter_app/lib/camera/frame_analyzer.dart
+40 -6
View File
@@ -1,3 +1,5 @@
import 'dart:typed_data';
import 'package:camera/camera.dart';
import '../detection/detection_result.dart';
@@ -44,7 +46,8 @@ class FrameAnalyzer {
int rotation,
int width,
int height,
int processMs) {
int processMs,
String yuvDiag) {
detectCalls++;
lastProcessMs = processMs;
viewModel.onFramesAnalyzed(
@@ -59,6 +62,7 @@ class FrameAnalyzer {
lastError: lastError,
framesReceived: framesReceived,
lastProcessMs: lastProcessMs,
yuvDiag: yuvDiag,
);
}
@@ -80,15 +84,45 @@ class FrameAnalyzer {
);
}
void analyze(CameraImage image, int rotationDegrees) {
/// 节流与 busy 丢帧判定（两入口共用）；通过后才允许投递
bool _canSend() {
framesReceived++;
final w = worker;
if (w == null) return;
if (w == null) return false;
final now = DateTime.now().millisecondsSinceEpoch;
if (now - _lastDetectMs < intervalMs) return;
if (now - _lastDetectMs < intervalMs) return false;
_lastDetectMs = now;
if (w.busy) return; // 上一帧未返回则丢帧，避免在途积压
w.analyze(image, rotationDegrees);
if (w.busy) return false; // 上一帧未返回则丢帧，避免在途积压
return true;
}
void analyze(CameraImage image, int rotationDegrees,
{bool rgbaOrder = false}) {
if (!_canSend()) return;
worker!.analyze(image, rotationDegrees, rgbaOrder: rgbaOrder);
}
/// 原生相机通道帧（Android）：字节已在 Kotlin 侧旋转成竖屏，rotation=0。
/// isBgra=true + rgbaOrder 与插件路径同语义：false=BGRA(rOff=2)/true=RGBA(rOff=0)
void analyzeRaw({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
required bool rgbaOrder,
int rotationDegrees = 0,
}) {
if (!_canSend()) return;
worker!.analyzeRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder,
rotationDegrees: rotationDegrees,
);
}
void reset() => _lastDetectMs = 0;
flutter_app/lib/detection/coordinate_mapper.dart
+4 -4
View File
@@ -12,7 +12,7 @@ class ViewRect {
double get centerY => (top + bottom) / 2;
}
/// 模型归一化坐标 → 预览视图坐标（含传感器旋转与 FIT_CENTER 裁剪）。
/// 模型归一化坐标 → 预览视图坐标（含传感器旋转与 FIT_COVER 全屏裁剪）。
class CoordinateMapper {
static ViewRect mapToView(
double normLeft,
@@ -53,10 +53,10 @@ class CoordinateMapper {
final portrait = rotation == 90 || rotation == 270;
final portW = portrait ? imageH : imageW;
final portH = portrait ? imageW : imageH;
// 3) FIT_CENTER 缩放与居中偏移
// 3) FIT_COVER 缩放与居中裁剪：放大到铺满视图，溢出部分裁掉
final scale = viewW / portW < viewH / portH
? viewW / portW
: viewH / portH;
? viewH / portH
: viewW / portW;
final offsetX = (viewW - portW * scale) / 2;
final offsetY = (viewH - portH * scale) / 2;
return ViewRect(
flutter_app/lib/detection/detector_worker.dart
+166 -14
View File
@@ -30,9 +30,10 @@ class DetectorWorker {
int _inFlight = 0;
bool _dead = false;
/// 结果回调：结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 / 处理耗时 ms
/// 结果回调：结果 / 运动区域 / 新颖区域 / 旋转角 / 图宽 / 图高 /
/// 处理耗时 ms / yuv 决策诊断串
void Function(List<DetectionResult>, List<MotionRegion>, List<MotionRegion>,
int, int, int, int)? onResult;
int, int, int, int, String)? onResult;
/// 单帧处理异常回调（不影响相机流）
void Function(String)? onError;
@@ -88,20 +89,37 @@ class DetectorWorker {
/// 是否忙（上一帧尚未返回）：忙则丢帧，避免在途积压
bool get busy => _inFlight > 0;
void analyze(CameraImage image, int rotationDegrees) {
void analyze(CameraImage image, int rotationDegrees, {bool rgbaOrder = false}) {
// 单平面 8888 判定：仅明确的 yuv420/nv21 走多平面 YUV 路径；
// bgra8888 与 unknown（插件未识别 RGBA_8888 输出时）都按 4 字节像素处理
final group = image.format.group;
analyzeRaw(
planes: image.planes.map((p) => p.bytes).toList(),
strides: image.planes.map((p) => p.bytesPerRow).toList(),
width: image.width,
height: image.height,
isBgra: group != ImageFormatGroup.yuv420 && group != ImageFormatGroup.nv21,
rgbaOrder: rgbaOrder,
rotationDegrees: rotationDegrees,
);
}
/// 原始字节帧投递（截屏注入用：toImage 的 RGBA 字节直接进检测，不经 CameraImage）
void analyzeRaw({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
required bool rgbaOrder,
required int rotationDegrees,
}) {
final port = _port;
if (port == null || _dead) return;
_inFlight++;
port.send([
'frame',
[
image.planes.map((p) => p.bytes).toList(),
image.planes.map((p) => p.bytesPerRow).toList(),
image.width,
image.height,
image.format.group == ImageFormatGroup.bgra8888,
rotationDegrees,
],
[planes, strides, width, height, isBgra, rotationDegrees, rgbaOrder],
]);
}
@@ -142,7 +160,7 @@ class DetectorWorker {
v[0] as double, v[1] as double, v[2] as double, v[3] as double))
.toList();
onResult?.call(dets, motion, novelty, list[1] as int, list[2] as int,
list[3] as int, list[7] as int);
list[3] as int, list[7] as int, list[8] as String);
break;
case 'log':
lastLog = list[1] as String;
@@ -151,6 +169,7 @@ class DetectorWorker {
case 'error':
_inFlight--;
onError?.call(list[1] as String);
break;
}
}
@@ -161,6 +180,13 @@ class DetectorWorker {
port.send(['reset']);
}
/// 调整置信度阈值（设置页滑块，worker 内实时生效）
void setMinScore(double v) {
final port = _port;
if (port == null || _dead) return;
port.send(['set-min-score', v]);
}
void dispose() {
_dead = true;
_isolate.kill(priority: Isolate.immediate);
@@ -179,6 +205,8 @@ Future<void> _workerMain(SendPort mainPort) async {
TfliteDetector? detector;
MotionDetector? motion;
BackgroundModel? background;
var lastDualMs = 0; // 双字节序推理诊断节流
Uint8List? prevY; // 上一帧 Y/RGBA 平面（帧间 diff 诊断）
await for (final msg in control) {
try {
final list = msg as List;
@@ -212,15 +240,104 @@ Future<void> _workerMain(SendPort mainPort) async {
final height = frame[3] as int;
final isBgra = frame[4] as bool;
final rotation = frame[5] as int;
final rgbaOrder = frame.length > 6 && (frame[6] as bool);
// 止血：非法帧（宽高/平面为空）直接丢弃并上报诊断，
// 避免下游组件越界（RGBA patch 后插件偶发 w/h=0 帧）
if (width <= 0 || height <= 0 || planes.isEmpty || planes[0].isEmpty) {
mainPort.send([
'error',
'bad frame w=$width h=$height planes=${planes.length} '
'p0=${planes.isNotEmpty ? planes[0].length : 0} '
'stride=${strides.isNotEmpty ? strides[0] : '-'} '
'bgra=$isBgra'
]);
break;
}
final sw = Stopwatch()..start();
// 帧内容统计（诊断）：Y/RGBA 平面 min/max/mean + 与上帧的平均绝对差。
// 均匀灰帧 → min≈max≈mean；静止灰帧 → diff≈0；真实画面 → 分布宽且 diff>0
final yPlane = planes[0];
var yMin = 255, yMax = 0, ySum = 0, diff = 0, sampled = 0;
final prev = prevY;
for (var i = 0; i < yPlane.length; i += 8) {
final v = yPlane[i];
if (v < yMin) yMin = v;
if (v > yMax) yMax = v;
ySum += v;
if (prev != null && i < prev.length) {
final d = v - prev[i];
diff += d < 0 ? -d : d;
}
sampled++;
}
prevY = yPlane;
final yMean = ySum / sampled;
final yDiff =
prev == null ? -1.0 : diff / (sampled * 255.0);
// UV 平面统计（诊断）：色序/值域异常会导致解码偏色
var uv1 = '-';
if (planes.length > 1) {
final u = planes[1];
var uMin = 255, uMax = 0, uSum = 0, n = 0;
for (var i = 0; i < u.length; i += 8) {
final v = u[i];
if (v < uMin) uMin = v;
if (v > uMax) uMax = v;
uSum += v;
n++;
}
uv1 = 's=${strides[1]} min=$uMin max=$uMax mean=${(uSum / n).toStringAsFixed(0)}';
if (planes.length > 2) {
final v2 = planes[2];
var vMin = 255, vMax = 0, vSum = 0, n2 = 0;
for (var i = 0; i < v2.length; i += 8) {
final v = v2[i];
if (v < vMin) vMin = v;
if (v > vMax) vMax = v;
vSum += v;
n2++;
}
uv1 += ' v:min=$vMin max=$vMax mean=${(vSum / n2).toStringAsFixed(0)}';
}
}
// 首帧（或相机重启后）自适应判定 YUV 值域与色序，再跑正式推理
if (!isBgra && !d.yuvModeKnown) {
d.decideYuvChroma(
planes: planes,
strides: strides,
width: width,
height: height,
);
}
var results = d.detectRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder,
);
// 自愈：判定后 1.5s 内无检测且帧可用 → 用实时帧重跑完整判定
// （首帧模糊/暗帧导致启发式猜错时，画面稳定后 oracle 可分胜负）
if (!isBgra && d.yuvModeKnown && !d.yuvRetried &&
results.length <= 1 &&
DateTime.now().millisecondsSinceEpoch - d.yuvDecisionMs > 1500 &&
d.retryDecision(
planes: planes,
strides: strides,
width: width,
height: height,
)) {
results = d.detectRaw(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
);
}
// 低分野鸡框过视觉先验（颜色/位置），减少户外误报
results = VisualPrior.filter(
results,
@@ -229,6 +346,7 @@ Future<void> _workerMain(SendPort mainPort) async {
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder,
);
final motionRegions = m.detectMotionRaw(
planes[0], strides[0], width, height);
@@ -236,6 +354,28 @@ Future<void> _workerMain(SendPort mainPort) async {
b.updateRaw(planes[0], strides[0], width, height);
sw.stop();
var dualDiag = '';
if (isBgra &&
DateTime.now().millisecondsSinceEpoch - lastDualMs > 3000) {
lastDualMs = DateTime.now().millisecondsSinceEpoch;
final a = d.diagnoseOrder(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: true,
rgbaOrder: false);
final b = d.diagnoseOrder(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: true,
rgbaOrder: true);
dualDiag = ' | dual BGRA:${a.$1}@${(a.$2 * 100).toStringAsFixed(1)}%'
' RGBA:${b.$1}@${(b.$2 * 100).toStringAsFixed(1)}%';
}
mainPort.send([
'result',
rotation,
@@ -252,14 +392,26 @@ Future<void> _workerMain(SendPort mainPort) async {
.map((mr) => [mr.left, mr.top, mr.right, mr.bottom])
.toList(),
sw.elapsedMilliseconds,
'planes=${planes.length} yLen=${yPlane.length} stride=${strides[0]} '
'y:min=$yMin max=$yMax mean=${yMean.toStringAsFixed(1)} '
'diff=${yDiff < 0 ? '-' : yDiff.toStringAsFixed(3)} '
'uv1:[$uv1] | ${d.yuvDiag}$dualDiag',
]);
break;
case 'reset':
motion?.reset();
background?.reset();
detector?.resetYuvMode();
break;
case 'set-min-score':
detector?.minScore = (list[1] as num).toDouble();
mainPort.send(['log', 'min-score=${detector?.minScore}']);
}
} catch (e) {
mainPort.send(['error', '$e']);
} catch (e, st) {
mainPort.send([
'error',
'$e\n${st.toString().split('\n').take(3).join('\n')}'
]);
}
}
}
flutter_app/lib/detection/tflite_detector.dart
+216 -29
View File
@@ -1,5 +1,6 @@
import 'dart:typed_data';
import 'package:flutter/foundation.dart' show debugPrint;
import 'package:tflite_flutter/tflite_flutter.dart';
import 'detection_result.dart';
@@ -11,8 +12,9 @@ import 'nms.dart';
/// 输入为 NCHW [1, 3, 704, 704]（litert 导出保留 torch 布局）。
class TfliteDetector {
static const int inputSize = 704;
// 野鸡数据置信度普遍偏低（0.1~0.2 量级），保留低分池供运动检测提升
static const double minScore = 0.10;
// 野鸡数据置信度普遍偏低（0.1~0.2 量级），保留低分池供运动检测提升；
// 可运行时调整（设置页滑块），默认 0.10
double minScore = 0.10;
static const double iouThreshold = 0.45;
static const int maxDetections = 20;
static const String modelAsset = 'assets/model.tflite';
@@ -75,13 +77,15 @@ class TfliteDetector {
required int width,
required int height,
required bool isBgra,
bool rgbaOrder = false,
}) {
preprocess(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra);
isBgra: isBgra,
rgbaOrder: rgbaOrder);
// 传原始字节视图而非 Float32List：tflite_flutter 会对非 ByteBuffer/Uint8List
// 输入调用 resizeInputTensor（1 维 [1486848]），使 node 0 TRANSPOSE prepare 失败
_interpreter.run(_input.buffer.asUint8List(), _output);
@@ -103,30 +107,36 @@ class TfliteDetector {
.toList();
}
/// 按像素格式分派：iOS bgra8888 单平面 / Android yuv420 多平面。
/// 按像素格式分派：单平面 RGBA/BGRA（iOS bgra8888 / Android 实验） / yuv420 多平面。
void preprocess({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
bool rgbaOrder = false,
}) {
if (isBgra) {
_preprocessBgra(planes[0], strides[0], width, height);
_preprocessBgra(planes[0], strides[0], width, height, rgbaOrder);
} else {
_preprocessYuv(planes, strides, width, height);
}
}
/// BGRA8888 单平面（iOS）：每像素 4 字节 [b,g,r,a]，双线性采样，
/// letterbox（等比缩到长边 704，短边黑边补 0，与 YOLO 训练一致）。
void _preprocessBgra(Uint8List src, int stride, int srcW, int srcH) {
/// 单平面 8888（iOS bgra8888 = [b,g,r,a]；Android 实验 RGBA_8888 = [r,g,b,a]）：
/// 每像素 4 字节，双线性采样，letterbox（等比缩到长边 704，短边黑边补 0）。
void _preprocessBgra(
Uint8List src, int stride, int srcW, int srcH, bool rgbaOrder) {
final plane = inputSize * inputSize;
final scale = inputSize / srcW < inputSize / srcH
? inputSize / srcW
: inputSize / srcH;
final dx = (inputSize - srcW * scale) / 2;
final dy = (inputSize - srcH * scale) / 2;
// rgbaOrder=false（iOS BGRA）: +0 B、+1 G、+2 R、+3 A；
// rgbaOrder=true（Android RGBA）: +0 R、+1 G、+2 B、+3 A
final rOff = rgbaOrder ? 0 : 2;
final bOff = rgbaOrder ? 2 : 0;
for (var oy = 0; oy < inputSize; oy++) {
final syf = (oy - dy) / scale;
@@ -153,23 +163,22 @@ class TfliteDetector {
final y1 = y0 < srcH - 1 ? y0 + 1 : y0;
final fx = sxf - x0, fy = syf - y0;
// BGRA 字节序：+0 B、+1 G、+2 R、+3 A
final i00 = y0 * stride + x0 * 4;
final i10 = y0 * stride + x1 * 4;
final i01 = y1 * stride + x0 * 4;
final i11 = y1 * stride + x1 * 4;
final r00 = src[i00 + 2].toDouble();
final r00 = src[i00 + rOff].toDouble();
final g00 = src[i00 + 1].toDouble();
final b00 = src[i00].toDouble();
final r10 = src[i10 + 2].toDouble();
final b00 = src[i00 + bOff].toDouble();
final r10 = src[i10 + rOff].toDouble();
final g10 = src[i10 + 1].toDouble();
final b10 = src[i10].toDouble();
final r01 = src[i01 + 2].toDouble();
final b10 = src[i10 + bOff].toDouble();
final r01 = src[i01 + rOff].toDouble();
final g01 = src[i01 + 1].toDouble();
final b01 = src[i01].toDouble();
final r11 = src[i11 + 2].toDouble();
final b01 = src[i01 + bOff].toDouble();
final r11 = src[i11 + rOff].toDouble();
final g11 = src[i11 + 1].toDouble();
final b11 = src[i11].toDouble();
final b11 = src[i11 + bOff].toDouble();
_input[p] = _bl(r00, r10, r01, r11, fx, fy) / 255.0;
_input[p + plane] = _bl(g00, g10, g01, g11, fx, fy) / 255.0;
@@ -179,25 +188,27 @@ class TfliteDetector {
}
/// letterbox 缩放 + YUV → RGB 归一化 0~1（NCHW），双线性采样。
/// 兼容 NV12（iOS 双平面，UV 交错）与 I420（Android 三平面）。
/// 兼容 NV12（双平面，UV 交错）与 I420（三平面）。
/// 首帧自适应：Y 值域（full/limited）与色序（U 先/V 先）因设备而异，
/// 静态假设会在部分机型上产生偏色 → 检测退化。
void _preprocessYuv(
List<Uint8List> planes, List<int> strides, int srcW, int srcH) {
final plane = inputSize * inputSize;
final y = planes[0];
final nv12 = planes.length == 2;
final uv = nv12 ? planes[1] : null;
final u = nv12 ? null : planes[1];
final v = nv12 ? null : planes[2];
final yStride = strides[0];
final uvStride = strides[1];
final vStride =
nv12 ? uvStride : (strides.length > 2 ? strides[2] : strides[1]);
// U/V 平面采样（nv12：偶位 U 奇位 V；i420：三平面分离）
// 色序修正后的 U/V 采样（nv12：偶位 U 奇位 V，NV21 相反；i420：平面 1/2 对调）
double uAt(int x, int y) => nv12
? uv![y * uvStride + x * 2] - 128.0
: u![y * uvStride + x] - 128.0;
? uv![y * uvStride + (_yuvSwapChroma ? x * 2 + 1 : x * 2)] - 128.0
: planes[_yuvSwapChroma ? 2 : 1][y * uvStride + x] - 128.0;
double vAt(int x, int y) => nv12
? uv![y * uvStride + x * 2 + 1] - 128.0
: v![y * uvStride + x] - 128.0;
? uv![y * uvStride + (_yuvSwapChroma ? x * 2 : x * 2 + 1)] - 128.0
: planes[_yuvSwapChroma ? 1 : 2][y * vStride + x] - 128.0;
final scale = inputSize / srcW < inputSize / srcH
? inputSize / srcW
@@ -256,10 +267,11 @@ class TfliteDetector {
final v11 = vAt(ux1, uy1);
final vv = _bl(v00, v10, v01, v11, fx, fy);
// 有限范围展开（VideoRange Y 16~235，Cb/Cr 16~240）
final yr = (yy - 16.0) * (255.0 / 219.0);
final un = uu * (255.0 / 224.0);
final vn = vv * (255.0 / 224.0);
// 值域展开：有限范围 VideoRange（Y 16~235，Cb/Cr 16~240）需线性拉伸；
// 全值域相机直接使用原始值（与 iOS bgra 一致）
final yr = _yuvFullRange ? yy : (yy - 16.0) * (255.0 / 219.0);
final un = _yuvFullRange ? uu : uu * (255.0 / 224.0);
final vn = _yuvFullRange ? vv : vv * (255.0 / 224.0);
// NCHW：r/g/b 分平面存储
_input[p] = (yr + 1.402 * vn) / 255.0;
@@ -269,6 +281,155 @@ class TfliteDetector {
}
}
/// 首帧自适应判定 YUV 模式，后续帧复用（相机重启后由 worker 复位重判）。
/// - 值域：有限范围黑电平恒为 16，低于 12 只可能是全值域。
/// - 色序：以模型本身为 oracle——同一帧按两种色序各推理一次，
/// 检测数/最高分/总分更高者为真；两序均无检测时退回亮区色相计数启发
/// （户外最亮区域为天空应偏蓝，若按默认 U 先序解出偏红则为 V 先序）。
/// - 首帧可能曝光未收敛（过暗/全黑），此时 oracle 与启发式都不可信，
/// 保持未判定状态等下一帧，避免在垃圾帧上锁死错误色序（真机零检测根因）。
bool _yuvFullRange = false;
bool _yuvSwapChroma = false;
bool _yuvModeKnown = false;
bool _yuvRetried = false;
int _yuvDecisionMs = 0;
String _yuvDiag = '';
bool get yuvModeKnown => _yuvModeKnown;
bool get yuvRetried => _yuvRetried;
int get yuvDecisionMs => _yuvDecisionMs;
String get yuvDiag => _yuvDiag;
void decideYuvChroma({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
bool force = false,
}) {
if (_yuvModeKnown && !force) return;
final (yMin, yMax, yMean) = _yStats(planes[0], strides[0], width, height);
_yuvFullRange = yMin < 12;
if (yMean < 30 || yMax < 170) {
// 曝光未稳定：保持未判定，下一帧重试；始终昏暗则维持默认（同旧版）
if (!_yuvModeKnown) {
_yuvDiag = '等稳定帧 mean=${yMean.toStringAsFixed(0)} max=$yMax';
}
return;
}
_yuvModeKnown = true;
_yuvDecisionMs = DateTime.now().millisecondsSinceEpoch;
final a = _runWithSwap(planes, strides, width, height, false);
final b = _runWithSwap(planes, strides, width, height, true);
var swap = false;
if (a.$1 != b.$1) {
swap = b.$1 > a.$1;
} else if (a.$2 != b.$2) {
swap = b.$2 > a.$2;
} else if (a.$3 != b.$3) {
swap = b.$3 > a.$3;
} else {
swap = _brightRegionLeansRed(planes, strides, width, height, yMax);
}
_yuvSwapChroma = swap;
_yuvDiag = 'full=$_yuvFullRange swap=$_yuvSwapChroma'
' cA=${a.$1} sA=${a.$2.toStringAsFixed(3)}'
' cB=${b.$1} sB=${b.$2.toStringAsFixed(3)}';
debugPrint('[yuv] $_yuvDiag mean=${yMean.toStringAsFixed(0)} max=$yMax');
}
/// 判定后持续无检测的自愈：用实时帧重跑完整判定（仅一次）。
/// 首帧模糊/暗帧导致启发式猜错时，等画面稳定后 oracle 即可分胜负。
/// 返回是否执行了重判（随后应重跑 detectRaw 取新结果）。
bool retryDecision({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
}) {
if (_yuvRetried || !_yuvModeKnown) return false;
final (_, yMax, yMean) = _yStats(planes[0], strides[0], width, height);
if (yMean < 30 || yMax < 170) return false; // 帧仍不可用
_yuvRetried = true;
decideYuvChroma(
planes: planes, strides: strides, width: width, height: height,
force: true);
return true;
}
/// 相机（重新）启动后复位，首帧重新判定
void resetYuvMode() {
_yuvModeKnown = false;
_yuvRetried = false;
_yuvDiag = '';
}
/// 采样统计 Y 值域：(min, max, mean)，步长 16px 约 3600 样本
(int, int, double) _yStats(Uint8List y, int yStride, int w, int h) {
var yMin = 255, yMax = 0;
var sum = 0, n = 0;
for (var j = 0; j < h; j += 16) {
final row = j * yStride;
for (var i = 0; i < w; i += 16) {
final v = y[row + i];
if (v < yMin) yMin = v;
if (v > yMax) yMax = v;
sum += v;
n++;
}
}
return (yMin, yMax, sum / n);
}
/// 按指定色序推理一次，返回 (检测数, 最高分, 总分)
(int, double, double) _runWithSwap(
List<Uint8List> planes, List<int> strides, int w, int h, bool swap) {
_yuvSwapChroma = swap;
preprocess(planes: planes, strides: strides, width: w, height: h,
isBgra: false);
_interpreter.run(_input.buffer.asUint8List(), _output);
final dets = postprocess();
var maxScore = 0.0, sumScore = 0.0;
for (final d in dets) {
sumScore += d.score;
if (d.score > maxScore) maxScore = d.score;
}
return (dets.length, maxScore, sumScore);
}
bool _brightRegionLeansRed(
List<Uint8List> planes, List<int> strides, int w, int h, int yMax) {
final y = planes[0];
final yStride = strides[0];
final uvStride = strides[1];
final nv12 = planes.length == 2;
final uv = nv12 ? planes[1] : null;
// 最亮带（maxY-40 以上），整体偏暗的场景也能拿到足量样本
final brightMin = yMax - 40;
var blue = 0, red = 0;
for (var j = 0; j < h; j += 8) {
final yrow = j * yStride;
for (var i = 0; i < w; i += 8) {
if (y[yrow + i] < brightMin) continue;
final cj = j ~/ 2, ci = i ~/ 2;
if (nv12) {
final c = cj * uvStride + ci * 2;
if (c + 1 >= uv!.length) continue;
if (uv[c] > 150) blue++;
if (uv[c + 1] > 150) red++;
} else {
final c = cj * uvStride + ci;
if (c >= planes[1].length || c >= planes[2].length) continue;
if (planes[1][c] > 150) blue++;
if (planes[2][c] > 150) red++;
}
}
}
// 亮区偏红多于偏蓝 → 当前 U/V 假设反了
return red > blue;
}
static double _bl(double a, double b, double c, double d, double fx,
double fy) =>
(1 - fx) * (1 - fy) * a + fx * (1 - fy) * b +
@@ -308,5 +469,31 @@ class TfliteDetector {
return kept.take(maxDetections).toList();
}
/// 诊断：按指定字节序推理一次，返回 (检测数, 最高分)。
/// 用于对比 BGRA/RGBA 两种顺序在同一帧上的检测差异（验证字节序与场景可达性）。
(int, double) diagnoseOrder({
required List<Uint8List> planes,
required List<int> strides,
required int width,
required int height,
required bool isBgra,
required bool rgbaOrder,
}) {
preprocess(
planes: planes,
strides: strides,
width: width,
height: height,
isBgra: isBgra,
rgbaOrder: rgbaOrder);
_interpreter.run(_input.buffer.asUint8List(), _output);
final dets = postprocess();
var maxScore = 0.0;
for (final d in dets) {
if (d.score > maxScore) maxScore = d.score;
}
return (dets.length, maxScore);
}
void dispose() => _interpreter.close();
}
flutter_app/lib/detection/visual_prior.dart
+15 -6
View File
@@ -34,12 +34,14 @@ class VisualPrior {
required int width,
required int height,
required bool isBgra,
required bool rgbaOrder,
}) {
if (results.isEmpty || width <= 0 || height <= 0) return results;
final kept = <DetectionResult>[];
for (final r in results) {
final lowConfPheasant = r.label == 'pheasant' && r.score < maxScore;
if (lowConfPheasant && _reject(r, planes, strides, width, height, isBgra)) {
if (lowConfPheasant &&
_reject(r, planes, strides, width, height, isBgra, rgbaOrder)) {
continue;
}
kept.add(r);
@@ -48,7 +50,7 @@ class VisualPrior {
}
static bool _reject(DetectionResult r, List<Uint8List> planes,
List<int> strides, int width, int height, bool isBgra) {
List<int> strides, int width, int height, bool isBgra, bool rgbaOrder) {
// 位置线索：detectRaw 输出为图像坐标系，centerY 直接可判天空区
if (r.centerY < skyTopRatio) return true;
@@ -63,7 +65,8 @@ class VisualPrior {
for (var gx = -2; gx <= 2; gx++) {
final px = (cx + gx * halfW / 2).round().clamp(0, width - 1).toInt();
final py = (cy + gy * halfH / 2).round().clamp(0, height - 1).toInt();
final (r_, g_, b_) = _pixel(planes, strides, px, py, width, height, isBgra);
final (r_, g_, b_) =
_pixel(planes, strides, px, py, width, height, isBgra, rgbaOrder);
total++;
final mn = math.min(r_, math.min(g_, b_));
final mx = math.max(r_, math.max(g_, b_));
@@ -80,14 +83,20 @@ class VisualPrior {
}
/// 读取单像素 RGB（0~255）。
/// BGRA 单平面：每像素 4 字节 [b,g,r,a]；
/// 8888 单平面按实际字节序取通道：BGRA=[b,g,r,a]（iOS 插件）、
/// RGBA=[r,g,b,a]（Android 自写原生通道）——字节序写死会让 Android
/// 低分框采样到 R/B 互换的颜色（橙色野鸡身被误判成"蓝色"）整批误杀；
/// YUV：y 平面 + 4:2:0 半分辨率 U/V（NV12 交错或 I420 分离）。
static (double, double, double) _pixel(List<Uint8List> planes,
List<int> strides, int x, int y, int width, int height, bool isBgra) {
List<int> strides, int x, int y, int width, int height, bool isBgra,
bool rgbaOrder) {
if (isBgra) {
final src = planes[0];
final i = y * strides[0] + x * 4;
return (src[i + 2].toDouble(), src[i + 1].toDouble(), src[i].toDouble());
final rOff = rgbaOrder ? 0 : 2;
final bOff = rgbaOrder ? 2 : 0;
return (src[i + rOff].toDouble(), src[i + 1].toDouble(),
src[i + bOff].toDouble());
}
final yy =
(planes[0][y * strides[0] + x] - 16.0) * (255.0 / 219.0);
flutter_app/pubspec.lock
+1 -1
View File
@@ -90,7 +90,7 @@ packages:
source: hosted
version: "0.11.4"
camera_android_camerax:
dependency: "direct main"
dependency: transitive
description:
name: camera_android_camerax
sha256: "8516fe308bc341a5067fb1a48edff0ddfa57c0d3cdcc9dbe7ceca3ba119e2577"
flutter_app/pubspec.yaml
+3 -1
View File
@@ -11,8 +11,10 @@ dependencies:
flutter:
sdk: flutter
cupertino_icons: ^1.0.8
# Android 已改用自写原生相机通道（android/ 下 CameraChannel.kt，分析帧
# 原生侧旋转成竖屏后回传），不再依赖 camera_android_camerax 插件；
# camera + camera_avfoundation 仅用于 iOS 路径
camera: ^0.11.0
camera_android_camerax: ^0.6.6
camera_avfoundation: ^0.9.17
tflite_flutter: ^0.11.0
flutter_secure_storage: ^9.2.0