初始化 observer 项目:纯代码,不含权重与训练数据

This commit is contained in:
2026-08-20 13:11:57 +08:00
commit b9934c996d
52 changed files with 4737 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
*.iml
.gradle/
local.properties
.idea/
build/
captures/
.externalNativeBuild/
.cxx/
.DS_Store
training/datasets/
training/venv/
*.pt
+80
View File
@@ -0,0 +1,80 @@
plugins {
alias(libs.plugins.android.application)
alias(libs.plugins.kotlin.android)
alias(libs.plugins.kotlin.compose)
}
android {
namespace = "com.example.observer"
compileSdk = 35
defaultConfig {
applicationId = "com.example.observer"
minSdk = 24
targetSdk = 35
versionCode = 1
versionName = "1.0"
ndk {
abiFilters += listOf("arm64-v8a", "armeabi-v7a")
}
}
buildTypes {
release {
isMinifyEnabled = true
isShrinkResources = true
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro",
)
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
kotlinOptions {
jvmTarget = "17"
}
buildFeatures {
compose = true
}
packaging {
resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
}
}
}
dependencies {
implementation(libs.androidx.core.ktx)
implementation(libs.androidx.lifecycle.runtime.ktx)
implementation(libs.androidx.lifecycle.viewmodel.compose)
implementation(libs.androidx.lifecycle.runtime.compose)
implementation(libs.androidx.activity.compose)
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.compose.ui)
implementation(libs.androidx.compose.ui.graphics)
implementation(libs.androidx.compose.ui.tooling.preview)
implementation(libs.androidx.compose.material3)
debugImplementation(libs.androidx.compose.ui.tooling)
implementation(libs.androidx.camera.core)
implementation(libs.androidx.camera.camera2)
implementation(libs.androidx.camera.lifecycle)
implementation(libs.androidx.camera.view)
implementation(libs.org.tensorflow.lite)
implementation(libs.org.tensorflow.lite.gpu)
implementation(libs.androidx.datastore.preferences)
implementation(libs.kotlinx.coroutines.android)
testImplementation(libs.junit)
}
+3
View File
@@ -0,0 +1,3 @@
# TensorFlow Lite
-keep class org.tensorflow.** { *; }
-dontwarn org.tensorflow.**
+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera.any" android:required="true" />
<application
android:name=".ObserverApp"
android:allowBackup="false"
android:icon="@drawable/ic_launcher"
android:label="@string/app_name"
android:supportsRtl="true"
android:theme="@style/Theme.Observer">
<activity
android:name=".MainActivity"
android:exported="true"
android:screenOrientation="portrait"
android:configChanges="orientation|screenSize|keyboardHidden">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
+4
View File
@@ -0,0 +1,4 @@
pheasant
hare
dove
fish
@@ -0,0 +1,17 @@
package com.example.observer
import android.os.Bundle
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import com.example.observer.ui.ObserverApp
class MainActivity : ComponentActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
enableEdgeToEdge()
super.onCreate(savedInstanceState)
setContent {
ObserverApp()
}
}
}
@@ -0,0 +1,26 @@
package com.example.observer
import android.app.Application
import android.content.Context
import com.example.observer.data.SettingsRepository
import com.example.observer.detection.Detector
import com.example.observer.detection.TFLiteDetector
import com.example.observer.distance.DistanceEstimator
import com.example.observer.reminder.Reminder
class ObserverApp : Application() {
lateinit var container: AppContainer
private set
override fun onCreate() {
super.onCreate()
container = AppContainer(this)
}
}
class AppContainer(context: Context) {
val settingsRepository = SettingsRepository(context)
val detector: Detector? = TFLiteDetector.create(context)
val distanceEstimator = DistanceEstimator(context)
val reminder = Reminder(context)
}
@@ -0,0 +1,82 @@
package com.example.observer.camera
import android.util.Log
import androidx.camera.core.Camera
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.Preview
import androidx.camera.camera2.interop.Camera2CameraInfo
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import androidx.lifecycle.LifecycleOwner
import java.util.concurrent.Executor
class CameraController(
private val lifecycleOwner: LifecycleOwner,
private val analysisExecutor: Executor,
) {
var onCameraIdChanged: ((String) -> Unit)? = null
var onInitError: (() -> Unit)? = null
var currentCameraId: String? = null
private set
private var cameraProvider: ProcessCameraProvider? = null
private var isFrontFacing = false
fun start(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
val future = ProcessCameraProvider.getInstance(previewView.context)
future.addListener({
try {
val provider = future.get()
cameraProvider = provider
bind(previewView, analyzer)
} catch (e: Exception) {
Log.e(TAG, "相机初始化失败", e)
onInitError?.invoke()
}
}, ContextCompat.getMainExecutor(previewView.context))
}
fun switchCamera(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
isFrontFacing = !isFrontFacing
bind(previewView, analyzer)
}
private fun bind(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
val provider = cameraProvider ?: return
provider.unbindAll()
val preview = Preview.Builder().build().also {
it.surfaceProvider = previewView.surfaceProvider
}
val imageAnalysis = ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build()
analyzer?.let { imageAnalysis.setAnalyzer(analysisExecutor, it) }
val camera: Camera = provider.bindToLifecycle(
lifecycleOwner,
currentSelector(),
preview,
imageAnalysis,
)
currentCameraId = try {
Camera2CameraInfo.from(camera.cameraInfo).cameraId
} catch (e: Exception) {
Log.w(TAG, "获取 cameraId 失败", e)
null
}
currentCameraId?.let { onCameraIdChanged?.invoke(it) }
}
private fun currentSelector(): CameraSelector =
if (isFrontFacing) CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA
companion object {
private const val TAG = "CameraController"
}
}
@@ -0,0 +1,38 @@
package com.example.observer.camera
import android.os.SystemClock
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import com.example.observer.detection.DetectionResult
import com.example.observer.detection.Detector
import com.example.observer.detection.MotionRegion
class FrameAnalyzer(
private val detector: Detector,
private val motionDetector: MotionDetector,
private val onResult: (results: List<DetectionResult>, rotation: Int, imageWidthPx: Int, imageHeightPx: Int, motionRegions: List<MotionRegion>) -> Unit,
) : ImageAnalysis.Analyzer {
@Volatile var intervalMs: Long = 100L
private var lastDetectMs = 0L
override fun analyze(imageProxy: ImageProxy) {
val now = SystemClock.elapsedRealtime()
if (now - lastDetectMs < intervalMs) {
imageProxy.close()
return
}
lastDetectMs = now
val bitmap = imageProxy.toBitmap()
val rotation = imageProxy.imageInfo.rotationDegrees
try {
val motionRegions = motionDetector.detectMotion(bitmap)
val results = detector.detect(bitmap)
onResult(results, rotation, bitmap.width, bitmap.height, motionRegions)
} finally {
imageProxy.close()
}
}
}
@@ -0,0 +1,60 @@
package com.example.observer.camera
import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Paint
import android.graphics.Rect
import com.example.observer.detection.MotionAggregator
import com.example.observer.detection.MotionRegion
import kotlin.math.min
/**
* 轻量运动检测:相邻帧灰度差分 + 分块聚合。
* 小尺寸工作(约 128x128 内),每帧开销亚毫秒级,在分析线程串行调用。
* 相机大幅移动时(全屏帧差)自动忽略本帧,避免误报。
*/
class MotionDetector(
private val maxWidth: Int = 128,
private val maxHeight: Int = 128,
) {
private val smallBitmap = Bitmap.createBitmap(maxWidth, maxHeight, Bitmap.Config.ARGB_8888)
private val canvas = Canvas(smallBitmap)
private val paint = Paint(Paint.FILTER_BITMAP_FLAG)
private var prevGray: IntArray? = null
private var grayCache: IntArray = IntArray(0)
fun detectMotion(bitmap: Bitmap): List<MotionRegion> {
val scale = min(maxWidth.toFloat() / bitmap.width, maxHeight.toFloat() / bitmap.height)
val w = (bitmap.width * scale).toInt()
val h = (bitmap.height * scale).toInt()
if (w == 0 || h == 0) return emptyList()
canvas.drawBitmap(bitmap, null, Rect(0, 0, w, h), paint)
if (grayCache.size != w * h) grayCache = IntArray(w * h)
smallBitmap.getPixels(grayCache, 0, w, 0, 0, w, h)
val prev = prevGray
prevGray = grayCache.copyOf()
if (prev == null || prev.size != grayCache.size) return emptyList()
// 取绿色通道近似亮度
val gray = IntArray(grayCache.size)
for (i in grayCache.indices) gray[i] = (grayCache[i] shr 8) and 0xFF
val prevGreen = IntArray(prev.size)
for (i in prev.indices) prevGreen[i] = (prev[i] shr 8) and 0xFF
val diff = MotionAggregator.diffMask(gray, prevGreen)
val motionTotal = diff.sum()
// 全屏大差异 → 相机移动/大范围变化,忽略本帧
if (motionTotal > w * h / 2) return emptyList()
if (motionTotal < 12) return emptyList()
return MotionAggregator.aggregate(diff, w, h)
}
/** 相机切换后重置参考帧,避免旧帧误差 */
fun reset() {
prevGray = null
}
}
@@ -0,0 +1,75 @@
package com.example.observer.data
import android.content.Context
import androidx.datastore.preferences.core.booleanPreferencesKey
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.floatPreferencesKey
import androidx.datastore.preferences.core.stringPreferencesKey
import androidx.datastore.preferences.preferencesDataStore
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
enum class DetectMode(val intervalMs: Long, val label: String) {
CONTINUOUS(100L, "连续"),
STANDARD(300L, "标准"),
POWER_SAVING(1000L, "省电"),
}
data class AppSettings(
val confThreshold: Float = 0.40f,
val habitatThreshold: Float = 0.35f,
val detectMode: DetectMode = DetectMode.CONTINUOUS,
val vibrateEnabled: Boolean = true,
val soundEnabled: Boolean = true,
val showDistance: Boolean = true,
val habitatAlertEnabled: Boolean = true,
)
private val Context.settingsDataStore by preferencesDataStore(name = "settings")
class SettingsRepository(private val context: Context) {
val settings: Flow<AppSettings> = context.settingsDataStore.data.map { prefs ->
AppSettings(
confThreshold = prefs[KEY_CONF] ?: 0.40f,
habitatThreshold = prefs[KEY_HABITAT] ?: 0.35f,
detectMode = DetectMode.entries.firstOrNull { it.name == prefs[KEY_MODE] }
?: DetectMode.CONTINUOUS,
vibrateEnabled = prefs[KEY_VIBRATE] ?: true,
soundEnabled = prefs[KEY_SOUND] ?: true,
showDistance = prefs[KEY_DISTANCE] ?: true,
habitatAlertEnabled = prefs[KEY_HABITAT_ALERT] ?: true,
)
}
suspend fun setConfThreshold(value: Float) =
context.settingsDataStore.edit { it[KEY_CONF] = value }
suspend fun setHabitatThreshold(value: Float) =
context.settingsDataStore.edit { it[KEY_HABITAT] = value }
suspend fun setDetectMode(mode: DetectMode) =
context.settingsDataStore.edit { it[KEY_MODE] = mode.name }
suspend fun setVibrateEnabled(value: Boolean) =
context.settingsDataStore.edit { it[KEY_VIBRATE] = value }
suspend fun setSoundEnabled(value: Boolean) =
context.settingsDataStore.edit { it[KEY_SOUND] = value }
suspend fun setShowDistance(value: Boolean) =
context.settingsDataStore.edit { it[KEY_DISTANCE] = value }
suspend fun setHabitatAlertEnabled(value: Boolean) =
context.settingsDataStore.edit { it[KEY_HABITAT_ALERT] = value }
companion object {
private val KEY_CONF = floatPreferencesKey("conf_threshold")
private val KEY_HABITAT = floatPreferencesKey("habitat_threshold")
private val KEY_MODE = stringPreferencesKey("detect_mode")
private val KEY_VIBRATE = booleanPreferencesKey("vibrate_enabled")
private val KEY_SOUND = booleanPreferencesKey("sound_enabled")
private val KEY_DISTANCE = booleanPreferencesKey("show_distance")
private val KEY_HABITAT_ALERT = booleanPreferencesKey("habitat_alert_enabled")
}
}
@@ -0,0 +1,54 @@
package com.example.observer.detection
/** 预览视图像素坐标矩形(纯 JVM 类型,便于单元测试) */
data class ViewRect(
val left: Float,
val top: Float,
val right: Float,
val bottom: Float,
) {
val width: Float get() = right - left
val height: Float get() = bottom - top
val centerX: Float get() = (left + right) / 2f
val centerY: Float get() = (top + bottom) / 2f
}
object CoordinateMapper {
/** 模型归一化坐标 → 预览视图坐标(含传感器旋转与 FIT_CENTER 裁剪) */
fun mapToView(
normLeft: Float,
normTop: Float,
normRight: Float,
normBottom: Float,
rotation: Int, // ImageProxy.imageInfo.rotationDegrees
imageW: Int,
imageH: Int, // 分析图像尺寸(横屏原图)
viewW: Int,
viewH: Int, // 预览视图尺寸
): ViewRect {
// 1) 旋转校正:图像方向 → 竖屏视图方向(归一化坐标)
val (x0, y0, x1, y1) = when (rotation) {
90 -> Quad(1 - normBottom, normLeft, 1 - normTop, normRight)
180 -> Quad(1 - normRight, 1 - normBottom, 1 - normLeft, 1 - normTop)
270 -> Quad(normTop, 1 - normRight, normBottom, 1 - normLeft)
else -> Quad(normLeft, normTop, normRight, normBottom)
}
// 2) 旋转后图像在竖屏方向上的尺寸
val portrait = rotation == 90 || rotation == 270
val portW = if (portrait) imageH else imageW
val portH = if (portrait) imageW else imageH
// 3) FIT_CENTER 缩放与居中偏移
val scale = minOf(viewW.toFloat() / portW, viewH.toFloat() / portH)
val offsetX = (viewW - portW * scale) / 2f
val offsetY = (viewH - portH * scale) / 2f
return ViewRect(
left = x0 * portW * scale + offsetX,
top = y0 * portH * scale + offsetY,
right = x1 * portW * scale + offsetX,
bottom = y1 * portH * scale + offsetY,
)
}
private data class Quad(val x0: Float, val y0: Float, val x1: Float, val y1: Float)
}
@@ -0,0 +1,21 @@
package com.example.observer.detection
data class DetectionResult(
val label: String,
val score: Float,
val left: Float,
val top: Float,
val right: Float,
val bottom: Float,
val distanceM: Float? = null,
) {
val width: Float get() = right - left
val height: Float get() = bottom - top
val centerX: Float get() = (left + right) / 2f
val centerY: Float get() = (top + bottom) / 2f
val isHabitat: Boolean get() = label == HABITAT_LABEL
companion object {
const val HABITAT_LABEL = "cover"
}
}
@@ -0,0 +1,8 @@
package com.example.observer.detection
import android.graphics.Bitmap
interface Detector {
/** 输入 RGBA 位图,输出归一化坐标检测结果(0..1) */
fun detect(bitmap: Bitmap): List<DetectionResult>
}
@@ -0,0 +1,103 @@
package com.example.observer.detection
import kotlin.math.abs
/**
* 帧差运动聚合(纯 JVM,可单测)。
* 输入:每像素 0/1 差分掩码,按 8x8 分块统计激活块,连通块聚合为运动区域。
*/
object MotionAggregator {
private const val BLOCK_GRID = 8
private const val BLOCK_ACTIVE_RATIO = 0.30f
private const val MAX_REGIONS = 3
fun aggregate(
diff: IntArray,
width: Int,
height: Int,
): List<MotionRegion> {
val bw = width / BLOCK_GRID
val bh = height / BLOCK_GRID
if (bw == 0 || bh == 0) return emptyList()
val active = BooleanArray(BLOCK_GRID * BLOCK_GRID)
for (by in 0 until BLOCK_GRID) {
for (bx in 0 until BLOCK_GRID) {
val blockW = if (bx == BLOCK_GRID - 1) width - bx * bw else bw
val blockH = if (by == BLOCK_GRID - 1) height - by * bh else bh
var motion = 0
for (y in by * bh until by * bh + blockH) {
var idx = y * width + bx * bw
for (x in 0 until blockW) {
motion += diff[idx + x]
}
idx += width
}
active[by * BLOCK_GRID + bx] = motion > blockW * blockH * BLOCK_ACTIVE_RATIO
}
}
// 连通块聚合(4 邻域)
val regions = ArrayList<MotionRegion>(MAX_REGIONS)
val visited = BooleanArray(active.size)
for (i in active.indices) {
if (!active[i] || visited[i]) continue
var minX = BLOCK_GRID
var minY = BLOCK_GRID
var maxX = -1
var maxY = -1
val stack = ArrayDeque<Int>()
stack.add(i)
visited[i] = true
while (stack.isNotEmpty()) {
val cur = stack.removeLast()
val bx = cur % BLOCK_GRID
val by = cur / BLOCK_GRID
minX = minOf(minX, bx)
maxX = maxOf(maxX, bx)
minY = minOf(minY, by)
maxY = maxOf(maxY, by)
for (nb in neighbors(cur)) {
if (active[nb] && !visited[nb]) {
visited[nb] = true
stack.add(nb)
}
}
}
if (maxX - minX > 3 || maxY - minY > 3) continue // 全屏噪声过滤
regions += MotionRegion(
left = minX * bw / width.toFloat(),
top = minY * bh / height.toFloat(),
right = ((maxX + 1) * bw).coerceAtMost(width) / width.toFloat(),
bottom = ((maxY + 1) * bh).coerceAtMost(height) / height.toFloat(),
)
if (regions.size >= MAX_REGIONS) break
}
return regions
}
private fun neighbors(i: Int): List<Int> {
val bx = i % BLOCK_GRID
val by = i / BLOCK_GRID
val list = ArrayList<Int>(4)
if (bx > 0) list += i - 1
if (bx < BLOCK_GRID - 1) list += i + 1
if (by > 0) list += i - BLOCK_GRID
if (by < BLOCK_GRID - 1) list += i + BLOCK_GRID
return list
}
/** 检测框中心是否落在运动区域内(用于置信度提升判定) */
fun centerInRegion(box: DetectionResult, region: MotionRegion): Boolean =
box.centerX in region.left..region.right && box.centerY in region.top..region.bottom
/** 帧差掩码生成(纯 JVM):|g - prev| > threshold → 1 */
fun diffMask(gray: IntArray, prev: IntArray, threshold: Int = 25): IntArray {
val diff = IntArray(gray.size)
for (i in gray.indices) {
diff[i] = if (abs(gray[i] - prev[i]) > threshold) 1 else 0
}
return diff
}
}
@@ -0,0 +1,12 @@
package com.example.observer.detection
/** 运动区域(归一化坐标) */
data class MotionRegion(
val left: Float,
val top: Float,
val right: Float,
val bottom: Float,
) {
val centerX: Float get() = (left + right) / 2f
val centerY: Float get() = (top + bottom) / 2f
}
@@ -0,0 +1,21 @@
package com.example.observer.detection
fun iou(a: DetectionResult, b: DetectionResult): Float {
val x0 = maxOf(a.left, b.left)
val y0 = maxOf(a.top, b.top)
val x1 = minOf(a.right, b.right)
val y1 = minOf(a.bottom, b.bottom)
if (x1 <= x0 || y1 <= y0) return 0f
val inter = (x1 - x0) * (y1 - y0)
val union = a.width * a.height + b.width * b.height - inter
return if (union <= 0f) 0f else inter / union
}
fun nms(boxes: List<DetectionResult>, iouThreshold: Float): List<DetectionResult> {
val sorted = boxes.sortedByDescending { it.score }
val kept = mutableListOf<DetectionResult>()
for (b in sorted) {
if (kept.none { iou(b, it) > iouThreshold }) kept += b
}
return kept
}
@@ -0,0 +1,131 @@
package com.example.observer.detection
import android.content.Context
import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Matrix
import org.tensorflow.lite.Interpreter
import org.tensorflow.lite.gpu.GpuDelegate
import java.nio.ByteBuffer
import java.nio.ByteOrder
/**
* YOLOv8n 端侧推理实现。
* 模型输出布局(ultralytics tflite 导出):[1, 4 + nc, 8400]
* cx/cy/w/h 已归一化,类别得分已过 sigmoid;按列平铺:index = c * 8400 + anchor。
*/
class TFLiteDetector private constructor(
private val interpreter: Interpreter,
private val labels: List<String>,
) : Detector {
@Volatile var confThreshold: Float = 0.40f
@Volatile var habitatThreshold: Float = 0.35f
private val scaledBitmap = Bitmap.createBitmap(INPUT_SIZE, INPUT_SIZE, Bitmap.Config.ARGB_8888)
private val scaleCanvas = Canvas(scaledBitmap)
private val scaleMatrix = Matrix()
private val pixels = IntArray(INPUT_SIZE * INPUT_SIZE)
private val inputBuffer: ByteBuffer =
ByteBuffer.allocateDirect(1 * INPUT_SIZE * INPUT_SIZE * 3 * 4)
.order(ByteOrder.nativeOrder())
private val outputFloats = FloatArray((4 + NUM_CLASSES) * NUM_ANCHORS)
override fun detect(bitmap: Bitmap): List<DetectionResult> {
preprocess(bitmap)
interpreter.run(inputBuffer, outputFloats)
return postprocess()
}
private fun preprocess(bitmap: Bitmap) {
// CENTER_CROP 缩放保持宽高比
val scale = maxOf(INPUT_SIZE.toFloat() / bitmap.width, INPUT_SIZE.toFloat() / bitmap.height)
val dx = (INPUT_SIZE - bitmap.width * scale) / 2f
val dy = (INPUT_SIZE - bitmap.height * scale) / 2f
scaleMatrix.reset()
scaleMatrix.setScale(scale, scale)
scaleMatrix.postTranslate(dx, dy)
scaleCanvas.drawBitmap(bitmap, scaleMatrix, null)
inputBuffer.rewind()
scaledBitmap.getPixels(pixels, 0, INPUT_SIZE, 0, 0, INPUT_SIZE, INPUT_SIZE)
for (p in pixels) {
inputBuffer.putFloat(((p shr 16) and 0xFF) / 255f)
inputBuffer.putFloat(((p shr 8) and 0xFF) / 255f)
inputBuffer.putFloat((p and 0xFF) / 255f)
}
}
private fun postprocess(): List<DetectionResult> {
val boxes = ArrayList<DetectionResult>(8)
for (a in 0 until NUM_ANCHORS) {
val cx = outputFloats[a]
val cy = outputFloats[NUM_ANCHORS + a]
val w = outputFloats[2 * NUM_ANCHORS + a]
val h = outputFloats[3 * NUM_ANCHORS + a]
var bestCls = 0
var bestScore = 0f
for (c in 0 until NUM_CLASSES) {
val s = outputFloats[(4 + c) * NUM_ANCHORS + a]
if (s > bestScore) {
bestScore = s
bestCls = c
}
}
val label = labels.getOrElse(bestCls) { "unknown" }
// cover 用独立阈值;动物类保留低分池,供运动检测提升
val threshold = if (label == DetectionResult.HABITAT_LABEL) habitatThreshold else MIN_SCORE
if (bestScore < threshold) continue
boxes += DetectionResult(
label = label,
score = bestScore,
left = (cx - w / 2f).coerceIn(0f, 1f),
top = (cy - h / 2f).coerceIn(0f, 1f),
right = (cx + w / 2f).coerceIn(0f, 1f),
bottom = (cy + h / 2f).coerceIn(0f, 1f),
)
}
return nms(boxes, IOU_THRESHOLD).take(MAX_DETECTIONS)
}
companion object {
const val INPUT_SIZE = 320
/** 动物类最低保留分数:低于此分不输出(低分候选由运动检测提升显示) */
const val MIN_SCORE = 0.20f
private const val NUM_CLASSES = 4
private const val NUM_ANCHORS = 8400
private const val IOU_THRESHOLD = 0.45f
private const val MAX_DETECTIONS = 20
private const val MODEL_ASSET = "model.tflite"
private const val LABELS_ASSET = "labels.txt"
/** 模型缺失或加载失败返回 null(App 降级为仅预览) */
fun create(context: Context): TFLiteDetector? {
return try {
val options = Interpreter.Options().apply {
setNumThreads(4)
try {
addDelegate(GpuDelegate())
} catch (_: Throwable) {
// GPU 不可用,回退 CPU
}
}
val interpreter = Interpreter(loadModelFile(context), options)
val labels = context.assets.open(LABELS_ASSET).bufferedReader().readLines()
TFLiteDetector(interpreter, labels)
} catch (e: Exception) {
null
}
}
private fun loadModelFile(context: Context): ByteBuffer {
val bytes = context.assets.open(MODEL_ASSET).use { it.readBytes() }
return ByteBuffer.allocateDirect(bytes.size).order(ByteOrder.nativeOrder())
.apply { put(bytes); rewind() }
}
}
}
@@ -0,0 +1,63 @@
package com.example.observer.distance
import android.content.Context
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraManager
import kotlin.math.roundToInt
/**
* 单目距离估计(针孔模型):距离 = 焦距px × 参考体型 / 框高px。
* 误差预期 ±30%(5~50m);鱼类目标受折射影响、生境区域按植被高度估算,均仅供参考。
*/
class DistanceEstimator(context: Context) {
private val cameraManager =
context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
private val focalPxCache = HashMap<String, Float>()
// 参考体型(米)
private val speciesSizeM = mapOf(
"pheasant" to 0.45f, // 身高
"hare" to 0.45f, // 身长
"dove" to 0.30f, // 体长
"fish" to 1.00f, // 典型可见体长(误差大)
"cover" to 0.50f, // 植被高度(水面区域误差大)
)
fun estimate(
label: String,
boxHeightNorm: Float,
imageHeightPx: Int,
cameraId: String?,
): Float? {
val realH = speciesSizeM[label] ?: return null
val boxH = boxHeightNorm * imageHeightPx
if (boxH < 8f) return null // 过小目标不估算
val focalPx = focalPx(imageHeightPx, cameraId)
if (focalPx <= 0f) return null
return (focalPx * realH / boxH).roundToInt().toFloat()
}
/** focal_px = focal_mm × (imageHeightPx / sensorHeightMm) */
private fun focalPx(imageHeightPx: Int, cameraId: String?): Float {
val key = "$cameraId:$imageHeightPx"
focalPxCache[key]?.let { return it }
val id = cameraId ?: return -1f
val value = try {
val c = cameraManager.getCameraCharacteristics(id)
val focalMm = c.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)
?.firstOrNull()
val sensor = c.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)
if (focalMm != null && sensor != null) {
focalMm * imageHeightPx / sensor.height
} else {
-1f
}
} catch (e: Exception) {
-1f
}
focalPxCache[key] = value
return value
}
}
@@ -0,0 +1,85 @@
package com.example.observer.overlay
import android.graphics.Paint
import androidx.compose.foundation.Canvas
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.PathEffect
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.graphics.nativeCanvas
import androidx.compose.ui.graphics.toArgb
import androidx.compose.ui.unit.dp
import com.example.observer.detection.CoordinateMapper
import com.example.observer.detection.DetectionResult
import kotlin.math.roundToInt
private val speciesColors = mapOf(
"pheasant" to Color(0xFFE53935),
"hare" to Color(0xFF1E88E5),
"dove" to Color(0xFF8E24AA),
"fish" to Color(0xFF00ACC1),
"cover" to Color(0xFFFDD835),
)
private val speciesLabels = mapOf(
"pheasant" to "野鸡",
"hare" to "野兔",
"dove" to "斑鸠",
"fish" to "",
"cover" to "疑似区域",
)
@Composable
fun DetectionOverlay(
results: List<DetectionResult>,
rotation: Int,
imageW: Int,
imageH: Int,
modifier: Modifier = Modifier,
) {
Canvas(modifier = modifier) {
val viewW = size.width.toInt()
val viewH = size.height.toInt()
val textPaint = Paint().apply {
isAntiAlias = true
textSize = 30f
}
results.forEach { r ->
val rect = CoordinateMapper.mapToView(
r.left, r.top, r.right, r.bottom,
rotation, imageW, imageH, viewW, viewH,
)
val color = speciesColors[r.label] ?: Color.White
val stroke = Stroke(
width = 6.dp.toPx(),
pathEffect = if (r.isHabitat) {
PathEffect.dashPathEffect(floatArrayOf(20.dp.toPx(), 12.dp.toPx()))
} else {
null
},
)
drawRect(
color = color,
topLeft = Offset(rect.left, rect.top),
size = Size(rect.width, rect.height),
style = stroke,
)
textPaint.color = color.toArgb()
val text = buildString {
append(speciesLabels[r.label] ?: r.label)
append(" ${(r.score * 100).toInt()}%")
r.distanceM?.let { append(" · 约${it.roundToInt()}m") }
}
val labelTop = (rect.top - textPaint.textSize - 6).coerceAtLeast(0f)
drawContext.canvas.nativeCanvas.drawText(
text,
rect.left.coerceAtLeast(0f) + 4f,
labelTop + textPaint.textSize,
textPaint,
)
}
}
}
@@ -0,0 +1,74 @@
package com.example.observer.reminder
import android.content.Context
import android.media.AudioManager
import android.media.ToneGenerator
import android.os.Build
import android.os.SystemClock
import android.os.VibrationEffect
import android.os.Vibrator
class Reminder(context: Context) {
@Volatile var vibrateEnabled: Boolean = true
@Volatile var soundEnabled: Boolean = true
@Volatile var habitatAlertEnabled: Boolean = true
private val vibrator: Vibrator? =
context.getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator
private val toneGenerator: ToneGenerator? = try {
ToneGenerator(AudioManager.STREAM_NOTIFICATION, 60)
} catch (e: RuntimeException) {
null
}
private var lastAlertLabel: String? = null
private var lastAlertAt = 0L
/** 同类目标 10s 内只提醒一次;生境区域(cover)提醒方式与动物检测区分 */
fun onDetected(label: String) {
val now = SystemClock.elapsedRealtime()
if (label == lastAlertLabel && now - lastAlertAt < 10_000) return
lastAlertAt = now
lastAlertLabel = label
val isHabitat = label == "cover"
if (isHabitat && !habitatAlertEnabled) return
if (vibrateEnabled) vibrate(isHabitat)
if (soundEnabled) playTone(isHabitat)
}
private fun vibrate(isHabitat: Boolean) {
val v = vibrator ?: return
if (Build.VERSION.SDK_INT >= 26) {
val effect = if (isHabitat) {
VibrationEffect.createWaveform(longArrayOf(0, 150, 80, 150), -1)
} else {
VibrationEffect.createOneShot(200, VibrationEffect.DEFAULT_AMPLITUDE)
}
v.vibrate(effect)
} else {
@Suppress("DEPRECATION")
v.vibrate(if (isHabitat) longArrayOf(0, 150, 80, 150) else longArrayOf(0, 200), -1)
}
}
private fun playTone(isHabitat: Boolean) {
val tone = toneGenerator ?: return
if (isHabitat) {
// 双短音:生境预警
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 120)
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 120)
} else {
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 200)
}
}
fun release() {
toneGenerator?.release()
}
}
@@ -0,0 +1,22 @@
package com.example.observer.ui
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import com.example.observer.ui.camera.CameraScreen
import com.example.observer.ui.settings.SettingsScreen
import com.example.observer.ui.theme.ObserverTheme
@Composable
fun ObserverApp() {
ObserverTheme {
var showSettings by rememberSaveable { mutableStateOf(false) }
if (showSettings) {
SettingsScreen(onBack = { showSettings = false })
} else {
CameraScreen(onOpenSettings = { showSettings = true })
}
}
}
@@ -0,0 +1,244 @@
package com.example.observer.ui.camera
import android.Manifest
import android.content.pm.PackageManager
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.camera.view.PreviewView
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Button
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalLifecycleOwner
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.core.content.ContextCompat
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import androidx.lifecycle.viewmodel.compose.viewModel
import com.example.observer.ObserverApp
import com.example.observer.camera.CameraController
import com.example.observer.camera.FrameAnalyzer
import com.example.observer.camera.MotionDetector
import com.example.observer.data.AppSettings
import com.example.observer.data.DetectMode
import com.example.observer.overlay.DetectionOverlay
import kotlinx.coroutines.launch
import java.util.concurrent.Executors
@Composable
fun CameraScreen(onOpenSettings: () -> Unit) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
val container = (context.applicationContext as ObserverApp).container
val viewModel: CameraViewModel = viewModel(
factory = remember { CameraViewModelFactory(container) },
)
val state by viewModel.state.collectAsStateWithLifecycle()
val settings by container.settingsRepository.settings.collectAsStateWithLifecycle(
initialValue = AppSettings(),
)
val scope = rememberCoroutineScope()
val executor = remember { Executors.newSingleThreadExecutor() }
val previewView = remember {
PreviewView(context).apply {
scaleType = PreviewView.ScaleType.FIT_CENTER
implementationMode = PreviewView.ImplementationMode.PERFORMANCE
}
}
val cameraController = remember { CameraController(lifecycleOwner, executor) }
val motionDetector = remember { MotionDetector() }
val analyzer = remember(container.detector) {
container.detector?.let { d ->
FrameAnalyzer(d, motionDetector) { results, rotation, w, h, motion ->
viewModel.onFramesAnalyzed(results, rotation, w, h, motion)
}
}
}
val interval by viewModel.analyzerIntervalMs.collectAsState()
var hasCameraPermission by remember {
mutableStateOf(
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED,
)
}
val permissionLauncher = rememberLauncherForActivityResult(
ActivityResultContracts.RequestPermission(),
) { granted -> hasCameraPermission = granted }
var retryKey by remember { mutableIntStateOf(0) }
var initFailed by remember { mutableStateOf(false) }
DisposableEffect(Unit) {
cameraController.onCameraIdChanged = { viewModel.onCameraIdChanged(it) }
onDispose { executor.shutdown() }
}
LaunchedEffect(interval) { analyzer?.intervalMs = interval }
LaunchedEffect(retryKey) {
cameraController.onInitError = { initFailed = true }
if (hasCameraPermission) cameraController.start(previewView, analyzer)
}
LaunchedEffect(Unit) {
if (!hasCameraPermission) permissionLauncher.launch(Manifest.permission.CAMERA)
}
Box(modifier = Modifier.fillMaxSize().background(Color.Black)) {
if (hasCameraPermission) {
AndroidView(factory = { previewView }, modifier = Modifier.fillMaxSize())
if (state.modelReady) {
DetectionOverlay(
results = state.results,
rotation = state.rotation,
imageW = state.imageWidthPx,
imageH = state.imageHeightPx,
modifier = Modifier.fillMaxSize(),
)
} else {
Box(
modifier = Modifier.fillMaxSize().padding(24.dp),
contentAlignment = Alignment.Center,
) {
Text(
"模型未加载:请将训练好的 model.tflite 放入 app/src/main/assets 后重新构建",
color = Color.White,
textAlign = TextAlign.Center,
)
}
}
if (initFailed) {
Column(
modifier = Modifier
.align(Alignment.Center)
.background(Color(0x99000000), RoundedCornerShape(12.dp))
.padding(24.dp),
horizontalAlignment = Alignment.CenterHorizontally,
) {
Text("相机初始化失败", color = Color.White)
Spacer(Modifier.height(8.dp))
TextButton(onClick = {
initFailed = false
retryKey++
}) { Text("重试") }
}
}
CameraTopBar(
detectMode = settings.detectMode,
onCycleMode = {
val next = DetectMode.entries[
(settings.detectMode.ordinal + 1) % DetectMode.entries.size
]
scope.launch { container.settingsRepository.setDetectMode(next) }
},
onSwitchCamera = { cameraController.switchCamera(previewView, analyzer) },
modifier = Modifier.align(Alignment.TopCenter),
)
CameraBottomBar(
vibrateEnabled = settings.vibrateEnabled,
onToggleVibrate = {
scope.launch {
container.settingsRepository.setVibrateEnabled(!settings.vibrateEnabled)
}
},
onOpenSettings = onOpenSettings,
modifier = Modifier.align(Alignment.BottomCenter),
)
} else {
PermissionGuide(
onRequest = { permissionLauncher.launch(Manifest.permission.CAMERA) },
modifier = Modifier.align(Alignment.Center),
)
}
}
}
@Composable
private fun CameraTopBar(
detectMode: DetectMode,
onCycleMode: () -> Unit,
onSwitchCamera: () -> Unit,
modifier: Modifier = Modifier,
) {
Row(
modifier = modifier
.fillMaxWidth()
.background(Color(0x99000000))
.padding(horizontal = 12.dp, vertical = 4.dp),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
TextButton(onClick = onCycleMode) {
Text("频率:${detectMode.label}", color = Color.White)
}
TextButton(onClick = onSwitchCamera) {
Text("切换摄像头", color = Color.White)
}
}
}
@Composable
private fun CameraBottomBar(
vibrateEnabled: Boolean,
onToggleVibrate: () -> Unit,
onOpenSettings: () -> Unit,
modifier: Modifier = Modifier,
) {
Row(
modifier = modifier
.fillMaxWidth()
.background(Color(0x99000000))
.padding(horizontal = 12.dp, vertical = 4.dp),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
TextButton(onClick = onToggleVibrate) {
Text(if (vibrateEnabled) "提醒:开" else "提醒:关", color = Color.White)
}
TextButton(onClick = onOpenSettings) {
Text("设置", color = Color.White)
}
}
}
@Composable
private fun PermissionGuide(onRequest: () -> Unit, modifier: Modifier = Modifier) {
Column(
modifier = modifier.padding(32.dp),
horizontalAlignment = Alignment.CenterHorizontally,
) {
Text(
"需要相机权限才能进行实时识别",
color = Color.White,
textAlign = TextAlign.Center,
)
Spacer(Modifier.height(16.dp))
Button(onClick = onRequest) { Text("授权相机") }
}
}
@@ -0,0 +1,186 @@
package com.example.observer.ui.camera
import android.os.SystemClock
import androidx.lifecycle.ViewModel
import androidx.lifecycle.ViewModelProvider
import androidx.lifecycle.viewModelScope
import com.example.observer.AppContainer
import com.example.observer.data.DetectMode
import com.example.observer.data.SettingsRepository
import com.example.observer.detection.DetectionResult
import com.example.observer.detection.MotionAggregator
import com.example.observer.detection.MotionRegion
import com.example.observer.detection.TFLiteDetector
import com.example.observer.distance.DistanceEstimator
import com.example.observer.reminder.Reminder
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import java.util.LinkedHashMap
data class CameraUiState(
val modelReady: Boolean = false,
val results: List<DetectionResult> = emptyList(),
val rotation: Int = 90,
val imageWidthPx: Int = 0,
val imageHeightPx: Int = 0,
)
class CameraViewModel(
private val detector: TFLiteDetector?,
private val distanceEstimator: DistanceEstimator,
private val reminder: Reminder,
settingsRepository: SettingsRepository,
) : ViewModel() {
private val _state = MutableStateFlow(CameraUiState(modelReady = detector != null))
val state: StateFlow<CameraUiState> = _state.asStateFlow()
private val _analyzerIntervalMs = MutableStateFlow(DetectMode.CONTINUOUS.intervalMs)
val analyzerIntervalMs: StateFlow<Long> = _analyzerIntervalMs.asStateFlow()
@Volatile
private var showDistance = true
@Volatile
private var confThreshold = 0.40f
@Volatile
private var cameraId: String? = null
private val tracks = LinkedHashMap<String, Track>()
init {
viewModelScope.launch {
settingsRepository.settings.collect { s ->
detector?.confThreshold = s.confThreshold
detector?.habitatThreshold = s.habitatThreshold
reminder.vibrateEnabled = s.vibrateEnabled
reminder.soundEnabled = s.soundEnabled
reminder.habitatAlertEnabled = s.habitatAlertEnabled
showDistance = s.showDistance
confThreshold = s.confThreshold
_analyzerIntervalMs.value = s.detectMode.intervalMs
}
}
}
fun onCameraIdChanged(id: String) {
cameraId = id
}
/** 帧分析回调(分析线程调用,需线程安全) */
fun onFramesAnalyzed(
results: List<DetectionResult>,
rotation: Int,
imageWidthPx: Int,
imageHeightPx: Int,
motionRegions: List<MotionRegion>,
) {
val now = SystemClock.elapsedRealtime()
val byKey = HashMap<String, DetectionResult>(results.size)
results.forEach { byKey[keyOf(it)] = it }
synchronized(this) {
// 过期目标清除(消失 2s 后移除)
val expired = tracks.entries
.filter { it.key !in byKey && now - it.value.lastSeenMs > 2000 }
.map { it.key }
expired.forEach { tracks.remove(it) }
byKey.forEach { (k, r) ->
val t = tracks[k]
tracks[k] = if (t == null) Track(now, now, r) else Track(t.firstSeenMs, now, r)
}
if (tracks.size > MAX_TRACKS) {
val oldest = tracks.entries.sortedBy { it.value.lastSeenMs }.take(tracks.size - MAX_TRACKS)
oldest.forEach { tracks.remove(it.key) }
}
// 目标停留 ≥ 0.5s 才展示;消失 2s 内不闪断
val visible = tracks.values
.filter { now - it.firstSeenMs >= 500 && now - it.lastSeenMs <= 2000 }
.filter { shouldShow(it.result, motionRegions) }
.map { t ->
val r = t.result
val boosted = r.score < confThreshold &&
motionRegions.any { MotionAggregator.centerInRegion(r, it) }
val distance = if (showDistance) {
distanceEstimator.estimate(r.label, r.height, imageHeightPx, cameraId)
} else {
null
}
r.copy(
score = if (boosted) minOf(r.score + MOTION_BOOST, 1f) else r.score,
distanceM = distance,
)
}
// 提醒:刚变为可见的目标(防重复由 Reminder 按类别 10s 控制)
tracks.values.forEach { t ->
if (!shouldShow(t.result, motionRegions)) return@forEach
val age = now - t.firstSeenMs
if (age in 500..2100 && now - t.lastSeenMs <= 300) {
reminder.onDetected(t.result.label)
}
}
_state.update {
it.copy(
results = visible,
rotation = rotation,
imageWidthPx = imageWidthPx,
imageHeightPx = imageHeightPx,
)
}
}
}
/**
* 显示判定:cover 已按生境阈值过滤;动物分低于阈值时,
* 仅当与运动区域重叠(低分候选)才提升显示。
*/
private fun shouldShow(r: DetectionResult, motionRegions: List<MotionRegion>): Boolean {
if (r.isHabitat) return true
if (r.score >= confThreshold) return true
if (r.score < TFLiteDetector.MIN_SCORE) return false
return motionRegions.any { MotionAggregator.centerInRegion(r, it) }
}
override fun onCleared() {
reminder.release()
}
private fun keyOf(r: DetectionResult): String =
"${r.label}:${(r.centerX * 10).toInt()}:${(r.centerY * 10).toInt()}"
private data class Track(
val firstSeenMs: Long,
val lastSeenMs: Long,
val result: DetectionResult,
)
companion object {
private const val MAX_TRACKS = 30
/** 低分目标与运动区域重叠时的置信度提升量 */
private const val MOTION_BOOST = 0.15f
}
}
class CameraViewModelFactory(private val container: AppContainer) : ViewModelProvider.Factory {
@Suppress("UNCHECKED_CAST")
override fun <T : ViewModel> create(modelClass: Class<T>): T {
if (modelClass.isAssignableFrom(CameraViewModel::class.java)) {
return CameraViewModel(
detector = container.detector as? TFLiteDetector,
distanceEstimator = container.distanceEstimator,
reminder = container.reminder,
settingsRepository = container.settingsRepository,
) as T
}
throw IllegalArgumentException("Unknown ViewModel class: ${modelClass.name}")
}
}
@@ -0,0 +1,109 @@
package com.example.observer.ui.settings
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilterChip
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Slider
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import com.example.observer.ObserverApp
import com.example.observer.data.AppSettings
import com.example.observer.data.DetectMode
import kotlinx.coroutines.launch
@OptIn(ExperimentalMaterial3Api::class)
@Composable
fun SettingsScreen(onBack: () -> Unit) {
val context = LocalContext.current
val repository = (context.applicationContext as ObserverApp).container.settingsRepository
val settings by repository.settings.collectAsStateWithLifecycle(initialValue = AppSettings())
val scope = rememberCoroutineScope()
Scaffold(
topBar = {
TopAppBar(
title = { Text("设置") },
navigationIcon = {
TextButton(onClick = onBack) { Text("返回") }
},
)
},
) { padding ->
Column(
modifier = Modifier
.padding(padding)
.fillMaxSize()
.verticalScroll(rememberScrollState())
.padding(16.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
Text("动物识别阈值:${"%.2f".format(settings.confThreshold)}")
Slider(
value = settings.confThreshold,
onValueChange = { scope.launch { repository.setConfThreshold(it) } },
valueRange = 0.2f..0.7f,
)
Text("生境区域阈值:${"%.2f".format(settings.habitatThreshold)}")
Slider(
value = settings.habitatThreshold,
onValueChange = { scope.launch { repository.setHabitatThreshold(it) } },
valueRange = 0.2f..0.7f,
)
Text("检测频率")
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
DetectMode.entries.forEach { mode ->
FilterChip(
selected = settings.detectMode == mode,
onClick = { scope.launch { repository.setDetectMode(mode) } },
label = { Text(mode.label) },
)
}
}
SwitchRow("震动提醒", settings.vibrateEnabled) {
scope.launch { repository.setVibrateEnabled(it) }
}
SwitchRow("提示音", settings.soundEnabled) {
scope.launch { repository.setSoundEnabled(it) }
}
SwitchRow("距离标注", settings.showDistance) {
scope.launch { repository.setShowDistance(it) }
}
SwitchRow("生境区域预警", settings.habitatAlertEnabled) {
scope.launch { repository.setHabitatAlertEnabled(it) }
}
}
}
}
@Composable
private fun SwitchRow(title: String, checked: Boolean, onCheckedChange: (Boolean) -> Unit) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
Text(title)
Switch(checked = checked, onCheckedChange = onCheckedChange)
}
}
@@ -0,0 +1,21 @@
package com.example.observer.ui.theme
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.darkColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.ui.graphics.Color
private val DarkColors = darkColorScheme(
primary = Color(0xFF66BB6A),
onPrimary = Color(0xFF00391F),
secondary = Color(0xFF80CBC4),
surface = Color(0xFF1C1C1C),
background = Color(0xFF101010),
onBackground = Color(0xFFE4E4E4),
onSurface = Color(0xFFE4E4E4),
)
@Composable
fun ObserverTheme(content: @Composable () -> Unit) {
MaterialTheme(colorScheme = DarkColors, content = content)
}
+21
View File
@@ -0,0 +1,21 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="108dp"
android:height="108dp"
android:viewportWidth="108"
android:viewportHeight="108">
<path
android:fillColor="#1B5E20"
android:pathData="M54,54m-54,0a54,54 0,1 1,108 0a54,54 0,1 1,-108 0" />
<path
android:fillColor="#00000000"
android:strokeColor="#FFFFFF"
android:strokeWidth="6"
android:pathData="M54,54m-20,0a20,20 0,1 1,40 0a20,20 0,1 1,-40 0" />
<path
android:strokeColor="#FFFFFF"
android:strokeWidth="4"
android:pathData="M54,22 L54,34 M54,74 L54,86 M22,54 L34,54 M74,54 L86,54" />
<path
android:fillColor="#FFFFFF"
android:pathData="M54,54m-5,0a5,5 0,1 1,10 0a5,5 0,1 1,-10 0" />
</vector>
+3
View File
@@ -0,0 +1,3 @@
<resources>
<string name="app_name">野视</string>
</resources>
+7
View File
@@ -0,0 +1,7 @@
<resources>
<style name="Theme.Observer" parent="android:Theme.Material.Light.NoActionBar">
<item name="android:windowBackground">@android:color/black</item>
<item name="android:statusBarColor">@android:color/transparent</item>
<item name="android:navigationBarColor">@android:color/black</item>
</style>
</resources>
@@ -0,0 +1,61 @@
package com.example.observer.detection
import org.junit.Assert.assertEquals
import org.junit.Test
class CoordinateMapperTest {
@Test
fun rotation90_center_mapsToViewCenter() {
// 横屏图像 1280x720,旋转 90 后竖屏显示 720x1280,视图 1080x1920(等比,无偏移)
val rect = CoordinateMapper.mapToView(
normLeft = 0.4f, normTop = 0.4f, normRight = 0.6f, normBottom = 0.6f,
rotation = 90, imageW = 1280, imageH = 720,
viewW = 1080, viewH = 1920,
)
assertEquals(540f, rect.centerX, 1f)
assertEquals(960f, rect.centerY, 1f)
}
@Test
fun rotation90_corner_appliesFitCenterOffset() {
// 视图 1080x2400,竖屏图像 720x1280scale=1.5,纵向偏移 (2400-1920)/2=240
val rect = CoordinateMapper.mapToView(
normLeft = 0f, normTop = 0f, normRight = 0.5f, normBottom = 0.5f,
rotation = 90, imageW = 1280, imageH = 720,
viewW = 1080, viewH = 2400,
)
// 旋转后:x0=1-bottom=0.5, y0=left=0, x1=1-top=1, y1=right=0.5
assertEquals(540f, rect.left, 1f) // 0.5 * 720 * 1.5
assertEquals(240f, rect.top, 1f) // 0 * 1280 * 1.5 + 240
assertEquals(1080f, rect.right, 1f) // 1 * 720 * 1.5
assertEquals(1200f, rect.bottom, 1f) // 0.5 * 1280 * 1.5 + 240
}
@Test
fun rotation0_keepsCoordinates() {
val rect = CoordinateMapper.mapToView(
normLeft = 0.2f, normTop = 0.3f, normRight = 0.5f, normBottom = 0.7f,
rotation = 0, imageW = 1080, imageH = 1920,
viewW = 1080, viewH = 1920,
)
assertEquals(216f, rect.left, 1f)
assertEquals(576f, rect.top, 1f)
assertEquals(540f, rect.right, 1f)
assertEquals(1344f, rect.bottom, 1f)
}
@Test
fun rotation180_flipsBothAxes() {
val rect = CoordinateMapper.mapToView(
normLeft = 0.2f, normTop = 0.2f, normRight = 0.4f, normBottom = 0.4f,
rotation = 180, imageW = 1080, imageH = 1920,
viewW = 1080, viewH = 1920,
)
// 翻转:x'=1-x, y'=1-y
assertEquals(648f, rect.left, 1f) // (1-0.4) * 1080
assertEquals(1152f, rect.top, 1f) // (1-0.4) * 1920
assertEquals(864f, rect.right, 1f) // (1-0.2) * 1080
assertEquals(1536f, rect.bottom, 1f) // (1-0.2) * 1920
}
}
@@ -0,0 +1,66 @@
package com.example.observer.detection
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
class MotionAggregatorTest {
// 96x64 图,8x8 块 → 每块 12x8 像素
@Test
fun noMotion_returnsEmpty() {
val diff = IntArray(96 * 64)
assertTrue(MotionAggregator.aggregate(diff, 96, 64).isEmpty())
}
@Test
fun singleBlockMotion_detectsRegion() {
val diff = IntArray(96 * 64)
// 块 (2,3)x 24..35, y 24..31,全部置 1
for (y in 24 until 32) {
for (x in 24 until 36) diff[y * 96 + x] = 1
}
val regions = MotionAggregator.aggregate(diff, 96, 64)
assertEquals(1, regions.size)
val r = regions[0]
assertTrue(r.left <= 24f / 96f && r.right >= 36f / 96f)
assertTrue(r.top <= 24f / 64f && r.bottom >= 32f / 64f)
}
@Test
fun twoSeparateMotions_detectsTwoRegions() {
val diff = IntArray(96 * 64)
for (y in 0 until 8) for (x in 0 until 12) diff[y * 96 + x] = 1
for (y in 48 until 64) for (x in 72 until 96) diff[y * 96 + x] = 1
val regions = MotionAggregator.aggregate(diff, 96, 64)
assertEquals(2, regions.size)
}
@Test
fun globalNoise_filteredOut() {
val diff = IntArray(96 * 64)
// 全屏散点噪声(随机块),但无连续连通域
val rnd = java.util.Random(42)
for (i in diff.indices) if (rnd.nextFloat() < 0.1f) diff[i] = 1
val regions = MotionAggregator.aggregate(diff, 96, 64)
assertTrue(regions.isEmpty())
}
@Test
fun centerInRegion_matches() {
val box = DetectionResult(
label = "hare",
score = 0.30f,
left = 0.2f,
top = 0.3f,
right = 0.4f,
bottom = 0.5f,
)
val region = MotionRegion(0.1f, 0.2f, 0.5f, 0.6f)
assertTrue(MotionAggregator.centerInRegion(box, region))
val outside = MotionRegion(0.6f, 0.7f, 0.9f, 0.9f)
assertFalse(MotionAggregator.centerInRegion(box, outside))
}
}
@@ -0,0 +1,50 @@
package com.example.observer.detection
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
class NmsTest {
private fun box(l: Float, t: Float, r: Float, b: Float, score: Float, label: String = "x") =
DetectionResult(label, score, l, t, r, b)
@Test
fun overlappingBoxes_keepHighestScore() {
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 0.8f)
val b = box(0.12f, 0.12f, 0.52f, 0.52f, 0.6f)
val result = nms(listOf(a, b), 0.45f)
assertEquals(1, result.size)
assertEquals(0.8f, result[0].score, 1e-6f)
}
@Test
fun separateBoxes_bothKept() {
val a = box(0.1f, 0.1f, 0.3f, 0.3f, 0.8f)
val b = box(0.7f, 0.7f, 0.9f, 0.9f, 0.6f)
val result = nms(listOf(a, b), 0.45f)
assertEquals(2, result.size)
}
@Test
fun lowScoreBox_suppressedByHigherScore() {
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 0.9f)
val b = box(0.1f, 0.1f, 0.5f, 0.5f, 0.5f)
val result = nms(listOf(b, a), 0.45f)
assertEquals(1, result.size)
assertTrue(result[0].score > 0.5f)
}
@Test
fun iou_nonOverlapping_isZero() {
val a = box(0.0f, 0.0f, 0.2f, 0.2f, 1f)
val b = box(0.8f, 0.8f, 1f, 1f, 1f)
assertEquals(0f, iou(a, b), 1e-6f)
}
@Test
fun iou_identicalBoxes_isOne() {
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 1f)
assertEquals(1f, iou(a, a), 1e-6f)
}
}
+5
View File
@@ -0,0 +1,5 @@
plugins {
alias(libs.plugins.android.application) apply false
alias(libs.plugins.kotlin.android) apply false
alias(libs.plugins.kotlin.compose) apply false
}
+318
View File
@@ -0,0 +1,318 @@
# Observer(野视)· 野生动物实时识别 Android App — 技术方案
| 项目 | 内容 |
| --- | --- |
| 文档版本 | v1.1 |
| 编写日期 | 2026-08-17 |
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照留存) |
| 适用产品 | 纯 Android 原生 App |
---
## 1. 项目概述
### 1.1 项目背景
野生动物观察爱好者在户外需要一款便携工具:打开手机相机,即可从实时画面中识别野鸡,通过检测框与提醒辅助快速发现。
### 1.2 项目目标
- 端侧实时目标检测,**全程离线可用**,不依赖网络;
- 野鸡实时框选识别,展示类别、置信度与**大致距离**;
- 检测到目标时通过震动 / 声音提醒用户,辅助快速发现;
- 纯 Android 原生 AppKotlin),兼容 Android 7.0+ 主流机型,中端机流畅运行;
- **不做拍照、记录、统计等留存功能**,专注实时识别这一核心体验。
### 1.3 名词术语
| 术语 | 说明 |
| --- | --- |
| ImageAnalysis | CameraX 的帧分析用例,用于逐帧回调图像数据 |
| TFLite | TensorFlow LiteGoogle 端侧推理框架 |
| YOLO | You Only Look Once,单阶段目标检测算法 |
| NMS | Non-Maximum Suppression,非极大值抑制 |
| GPU Delegate | TFLite 的 GPU 加速委托,将算子下发 GPU 执行 |
| mAP | mean Average Precision,目标检测平均精度指标 |
| IoU | Intersection over Union,交并比 |
---
## 2. 需求概述
### 2.1 目标用户
| 用户群 | 典型诉求 |
| --- | --- |
| 野生动物观察爱好者 | 户外实时识别画面中的野鸡,不惊扰、近距离观察 |
| 户外徒步 / 摄影人群 | 快速发现野鸡,辅助取景构图 |
| 自然教育、科普工作者 | 物种识别辅助教学 |
### 2.2 核心场景
| 编号 | 场景 | 描述 |
| --- | --- | --- |
| S1 | 野外实时识别 | 徒步 / 观鸟时打开相机,实时框选画面中的野鸡,显示类别、置信度与大致距离 |
| S2 | 快速扫视寻找 | 移动取景快速扫视,检测到野鸡立即震动 / 声音提醒,无需停留操作 |
### 2.3 功能需求摘要
实时识别(P0)、识别结果叠加展示(P0)、检测提醒(P1)、生境区域预警(P1)、设置(P1)、合规提示(P0)、低光增强(P2)。详见《项目功能文档》。
### 2.4 非功能需求摘要
| 指标 | 目标 |
| --- | --- |
| 识别延迟 | 中端机单帧 ≤ 80ms |
| 预览流畅度 | 连续模式 2~3 帧检测一次,预览不卡顿 |
| 耗电 | 连续使用 1 小时耗电 ≤ 15% |
| 兼容性 | Android 7.0+minSdk 24),覆盖主流国产机与三星 |
| 稳定性 | 崩溃率 ≤ 0.5%,启动成功率 ≥ 99% |
---
## 3. 总体架构
### 3.1 架构图
```mermaid
graph TD
subgraph UI层["UI 层 · Jetpack Compose"]
MainScreen["相机主界面<br/>预览 + 检测叠加层"]
SettingsScreen["设置界面"]
end
subgraph ViewModel层["ViewModel 层"]
CameraViewModel["CameraViewModel"]
SettingsViewModel["SettingsViewModel"]
end
subgraph 领域层["领域层"]
Detector接口["Detector 接口"]
end
subgraph 基础设施层["基础设施层"]
CameraX["CameraX<br/>Preview / ImageAnalysis"]
TFLite["TFLite 推理引擎<br/>YOLOv8n 模型 + GPU Delegate"]
DataStore["DataStore 设置存储"]
end
UI层 --> ViewModel层
ViewModel层 --> 领域层
领域层 --> 基础设施层
```
### 3.2 分层说明
| 层 | 职责 |
| --- | --- |
| UI 层 | Compose 页面渲染、检测叠加层绘制、交互 |
| ViewModel 层 | 页面状态管理、识别结果分发、提醒触发 |
| 领域层 | 检测器抽象接口 |
| 基础设施层 | CameraX、TFLite、DataStore 等能力实现 |
### 3.3 核心数据流(识别链路)
```mermaid
flowchart LR
A["CameraX 相机帧"] --> B["ImageAnalysis 帧分析"]
B --> C["预处理<br/>缩放 320×320 / 归一化 / 旋转校正"]
C --> D["TFLite 推理<br/>YOLOv8n"]
D --> E["后处理<br/>解码 / NMS / 阈值过滤"]
E --> F["叠加层渲染<br/>边框 + 类别 + 置信度"]
F --> G["UI 展示"]
E --> H["结果提醒<br/>震动 / 提示音"]
```
### 3.4 部署形态
- **本地优先**:识别全部离线完成,运行期无网络请求;
- **云端(V2 可选)**:仅用于模型版本更新下发。
---
## 4. 技术选型
| 类别 | 选型 | 理由 |
| --- | --- | --- |
| 开发语言 | Kotlin | Android 官方推荐,协程生态成熟 |
| UI 框架 | Jetpack ComposeMaterial 3 | 声明式 UI,开发效率高,叠加层绘制灵活 |
| 相机 | CameraXPreview / ImageAnalysis | Jetpack 官方库,生命周期安全,机型兼容性最佳 |
| 目标检测模型 | YOLOv8n(自定义 4 类训练) | 精度/速度均衡,端侧部署方案成熟 |
| 推理框架 | TensorFlow Lite 2.16+GPU Delegate | 官方支持 GPU 加速;备选 NCNN / MNN |
| 配置存储 | DataStore Preferences | 阈值、开关等设置 |
| 异步 | Kotlin Coroutines + Flow | 主线程安全,生命周期感知 |
| 构建 | GradleKotlin DSL+ AGP 8.x | 现代构建配置 |
---
## 5. 核心功能技术方案
### 5.1 实时识别管线
- CameraX 组合:`Preview`(取景)+ `ImageAnalysis`(识别);
- `ImageAnalysis` 使用 `STRATEGY_KEEP_ONLY_LATEST` 背压策略,保证不积压帧;
- 输出格式使用 `OUTPUT_IMAGE_FORMAT_RGBA_8888`CameraX 1.3+),免去 YUV→RGB 手动转换;
- **帧节流**:连续模式每 2~3 帧检测一次;标准模式 300ms 一次;省电模式每秒一次(依据设置);
- 推理在独立检测线程执行,单例互斥锁防止重叠推理;分析线程不阻塞主线程。
### 5.2 检测叠加层
- Compose Canvas 绘制检测框(矩形 + 类别标签 + 置信度),颜色按类别区分;
- 检测框内同时标注**大致距离**(如"约 25m"),随检测框实时更新;
- 距离标注与检测框同生共灭,展示逻辑一致(≥ 0.5s 出现、消失 2s 后移除);
- 坐标映射链路:模型归一化坐标 → 旋转校正(sensor rotation)→ 预览视图坐标(含 FIT_CENTER 裁剪偏移修正);
- 目标保持 ≥ 0.5s 才展示,避免单帧误检闪烁。
### 5.3 距离标注(单目估计)
- 采用单目针孔模型:`距离 ≈ 焦距px × 物种参考体型 / 检测框像素高度`
- 焦距 px 由相机内参换算(`LENS_INFO_AVAILABLE_FOCAL_LENGTHS` × `SENSOR_INFO_PHYSICAL_SIZE`),个别机型回退用视场角推算;
- 物种参考体型内置表(见 6.1),每类一个平均体型值;
- 展示为"约 X m"5~50m 范围误差预期 ≤ ±30%;
- 纯算术运算,无额外模型与算力开销;V2 可引入地面平面法(设备俯仰角 + 相机高度)提升精度。
### 5.4 检测提醒
- 检测到目标时触发震动 / 提示音,用户可开关;
- 防重复打扰:10s 内同类目标只提醒一次;
- 目标消失后 2s 内不闪断显示,避免频繁提醒。
### 5.5 低光增强(P2
- 简单直方图均衡 / 亮度增益提升低光帧可见度;
- V2 可引入夜视增强网络思路。
---
## 6. AI 模型方案(核心)
### 6.1 目标类别定义
| 类别 ID | 英文标签 | 中文名 | 涵盖范围 | 参考体型(距离估计用) |
| --- | --- | --- | --- | --- |
| 0 | pheasant | 野鸡 | 环颈雉等雉类 | ≈ 0.45m(身高) |
### 6.2 模型选型对比
| 模型 | 输入尺寸 | 中端机单帧耗时(GPU) | 精度 | 说明 |
| --- | --- | --- | --- | --- |
| **YOLOv8n(推荐)** | 320~640 | 约 30~80ms | 高 | 支持自定义类别训练,精度/速度平衡 |
| EfficientDet-Lite0/2 | 320 | 约 20~50ms | 中 | 仅 COCO 预置类别 |
| YOLOv5s | 640 | 约 80~150ms | 高 | 体积与耗电偏大 |
| SSD MobileNetV2 | 300 | 约 15~30ms | 低 | 小目标与远距离效果差 |
**结论**:选用 **YOLOv8n**,使用预训练权重迁移学习自定义训练,导出 TFLite 端侧部署。
### 6.3 数据集方案
| 项 | 方案 |
| --- | --- |
| 数据量 | 1000~2000 张(首版可 500+ 起步,滚动补充) |
| 多样性 | 覆盖不同季节、晨昏/正午/逆光、远近距离、姿态、遮挡、背景(草丛/农田/林地/雪地) |
| 标注 | Roboflow 或 labelImgYOLO 格式(class, cx, cy, w, h |
| 数据增强 | Mosaic、MixUp、HSV 扰动、随机翻转、随机缩放裁剪 |
| 数据划分 | train 80% / val 10% / test 10% |
| 负样本 | 补充无目标场景图,控制误检 |
### 6.4 训练方案
- 框架:ultralytics YOLOv8,预训练权重 `yolov8n.pt` 迁移学习;
- 超参:imgsz=640(训练)、epochs 100~200(早停)、batch 16~32(视 GPU 而定);
- 评估指标:
- mAP@0.5 ≥ **0.85**(达标线 0.80);
- mAP@0.5:0.95 ≥ **0.55**
- 负样本误检率 ≤ **2%**
- 远距离小目标(高度 ≤ 20px)召回率 ≥ **60%**
### 6.5 模型转换与量化
- 导出命令:`yolo export model=best.pt format=tflite imgsz=320`
- 量化策略:
- 优先 **fp16**(配合 GPU Delegate,精度损失小);
- 低端机 / CPU 场景使用 **int8** 量化(体积更小、CPU 更快);
- 输入尺寸:320×320(连续检测)/ 416×416(远距离模式,V2)。
### 6.6 推理优化
- GPU Delegate 优先,初始化失败自动回退 CPU(NNAPI 可选);
- 推理线程 4 线程,首次推理前执行预热(dummy run);
- 输入输出 ByteBuffer / Bitmap 复用,避免热路径频繁分配。
### 6.7 精度与速度目标
| 指标 | 目标 |
| --- | --- |
| 检测阈值(默认) | confidence ≥ 0.40IoU-NMS = 0.45 |
| 单帧推理延迟 | ≤ 80ms(中端机,320 输入,GPU |
| 模型体积 | fp16 ≤ 8MBint8 ≤ 4MB |
| 识别类别数 | 1 类(野鸡) |
| 距离标注 | "约 X m"5~50m 误差 ≤ ±30%,计算开销可忽略 |
---
## 7. 性能指标与优化
| 指标 | 目标值 | 主要优化手段 |
| --- | --- | --- |
| 启动到相机预览 | ≤ 1.5s | 启动即初始化 CameraProvider,懒加载非核心模块 |
| 单帧检测延迟 | ≤ 80ms | GPU Delegate、320 输入、线程复用 |
| 预览帧率 | ≥ 25fps | 帧节流、KEEP_ONLY_LATEST、避免主线程工作 |
| 内存峰值 | ≤ 200MB | Bitmap/ByteBuffer 复用,无大对象常驻 |
| 连续 1 小时耗电 | ≤ 15% | 检测帧节流、省电模式、后台自动释放相机 |
| App 体积 | ≤ 40MB | ABI 拆分、模型量化、R8 混淆 |
---
## 8. 数据与隐私
- 无账号、无埋点、无广告 SDK,**不采集任何用户数据**;
- 不保存照片、不记录位置,运行期无网络请求;
- 模型与标签随 APK 打包,识别数据仅存在于内存中,随进程结束自动释放。
---
## 9. 安全与合规
- 产品定位为**野生动物观察、识别工具**,不提供任何猎捕、诱捕、伤害野生动物的功能或指导;
- 遵守《中华人民共和国野生动物保护法》《陆生野生动物保护实施条例》等法律法规;
- 野鸡(雉类)属"三有"保护动物,猎捕须依法取得许可,App 不鼓励、不协助非法猎捕;
- App 内置观察伦理提示:保持安全距离、不惊扰动物、遵守保护区管理规定(首次启动展示,功能文档 F07)。
---
## 10. 开发计划(里程碑)
| 阶段 | 周期 | 交付内容 |
| --- | --- | --- |
| M0 准备 | 1 周 | 需求确认、数据集启动采集、Android 工程脚手架、模型基线 |
| M1 MVP | 3~4 周 | 相机预览 + 实时检测 + 叠加层 + 检测提醒 + 设置 + 合规提示 |
| M2 打磨发布 | 1~2 周 | 性能优化、真机矩阵测试、混淆加固、上架准备 |
总计约 **5~7 周**(不含数据采集并行时间)。
---
## 11. 风险与应对
| 风险 | 影响 | 应对 |
| --- | --- | --- |
| 训练数据不足 / 类别相近 | 精度低、误检 | 持续滚动采集数据;增加难例挖掘;灰度发布迭代模型 |
| 小目标(远处动物) | 漏检 | 提高输入分辨率;帧节流换取算力;提示用户靠近/变焦;V2 引入 SAHI/tiling 或专用小目标模型 |
| 低光 / 夜间场景 | 漏检 | 低光增强预处理;V2 夜视模式 |
| 单目距离估计精度有限 | 距离显示不准 | 标注为"约"并明示误差预期;体型表持续校准;V2 地面平面法 + 水下折射修正 |
| 生境区域误报偏多 | 提醒频繁、体验差 | 独立低阈值 + 区分提醒方式 + 防重复机制;生境阈值可调;模型迭代降低误报 |
| 中低端机型性能不足 | 帧率低、发热 | 动态分辨率、降频检测、GPU 回退策略、省电模式 |
| CameraX 个别机型异常 | 黑屏/闪退 | 机型兼容测试矩阵、崩溃监控、失败回退(重试/默认配置) |
| 合规风险(被用于非法猎捕) | 法律风险 | 产品定位为观察工具、内置合规提示、不提供猎捕辅助功能 |
---
## 12. 验收标准
1. 野鸡测试集 mAP@0.5 ≥ 0.85,负样本误检率 ≤ 2%;
2. 中端机(如骁龙 7 系)单帧检测 ≤ 80ms,预览流畅无卡顿;
3. 连续使用 1 小时耗电 ≤ 15%,无异常发热;
4. 主流机型(小米 / 华为 / OPPO / vivo / 三星,各 ≥ 2 台)启动成功率 ≥ 99%,崩溃率 ≤ 0.5%;
5. 飞行模式下实时识别全流程(预览、检测、提醒、设置)可用;
6. 检测框正确显示类别、置信度与距离;5~50m 范围距离误差 ≤ ±30%;
7. 通过应用商店合规审核(隐私政策、权限声明完整)。
+193
View File
@@ -0,0 +1,193 @@
# Observer(野视)· 项目功能文档
| 项目 | 内容 |
| --- | --- |
| 文档版本 | v1.1 |
| 编写日期 | 2026-08-17 |
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照留存) |
| 配套文档 | 《01-技术方案》《03-技术实现文档》 |
---
## 1. 产品概述
### 1.1 产品定位
一款**纯 Android 原生**的野生动物实时识别 App。用户打开相机,即可从实时画面中识别野鸡,通过彩色检测框(含类别、置信度、距离标注)与震动 / 声音提醒辅助快速发现。**无拍照、无记录、无统计**,专注实时识别这一核心体验,**全程离线可用**。
### 1.2 目标用户
- 野生动物观察爱好者(观鸟 / 观兽);
- 户外徒步、摄影人群;
- 自然教育、科普工作者。
### 1.3 价值主张
- **即时发现**:打开相机即识别,检测到目标立即提醒,免去翻图鉴、猜物种;
- **零门槛**:不依赖网络,不依赖外设,一部手机即可;
- **纯净体验**:无留存功能、无账号、无广告,即开即用。
---
## 2. 功能架构(功能树)
```
Observer
├── 实时识别
│ ├── 相机实时预览(F01)
│ ├── 目标实时检测(F02)
│ ├── 识别结果展示与距离标注(F03)
│ ├── 检测提醒(F04
│ └── 低光增强(F06,P2)
├── 设置
│ ├── 识别参数(F05
│ └── 提醒设置(F04 子项)
└── 合规提示(F07
```
---
## 3. 功能需求清单
优先级定义:**P0**=必须,**P1**=V1.0**P2**=后续版本。
| 编号 | 功能 | 功能描述 | 优先级 |
| --- | --- | --- | --- |
| F01 | 相机实时预览 | 全屏取景,默认后置,支持自动对焦、双指缩放、点击对焦、前后摄切换 | P0 |
| F02 | 目标实时检测 | 对野鸡实时检测框选 | P0 |
| F03 | 识别结果展示与距离标注 | 检测框 + 类别名 + 置信度百分比 + 大致距离(约 X m);四类目标使用不同颜色边框 | P0 |
| F04 | 检测提醒 | 检测到目标时震动 / 提示音;支持开关与 10s 防重复 | P1 |
| F05 | 设置 | 置信度阈值(0.2~0.7)、检测频率(连续 / 标准 / 省电)、提醒开关、低光增强开关 | P1 |
| F06 | 低光增强 | 低光环境下自动亮度 / 对比度增强,提高识别率 | P2 |
| F07 | 合规提示 | 首次启动展示《观察伦理与法律提示》,需用户确认 | P0 |
| F08 | 生境区域预警 | 画面中未检测到目标时,识别草丛 / 灌木 / 水面等疑似生境区域,以黄色虚线框展示(含距离标注)并触发预警(独立阈值与提醒方式) | P1 |
---
## 4. 核心业务流程
### 4.1 首次启动与授权
```
启动 App → 合规提示页(F07,用户确认)
→ 请求相机权限
├─ 授权 → 进入相机主界面
└─ 拒绝 → 引导页说明用途,可重新授权
```
### 4.2 实时识别
```
相机主界面(默认后置)
→ 画面持续检测(连续 / 标准 / 省电频率)
→ 检测到目标:显示彩色检测框 + 类别 + 置信度 + 距离
├─ 触发提醒(震动 / 声音,10s 防重复)
└─ 目标停留 ≥ 0.5s 保持显示,消失后 2s 内不闪断
→ 未检测到目标时:识别疑似生境区域(草丛 / 灌木 / 水面)
├─ 黄色虚线框展示,标注"疑似区域"与距离
└─ 触发预警(提醒方式与动物检测区分)
→ 用户可点击检测框查看识别详情(物种、置信度、时间)
```
---
## 5. 界面设计
### 5.1 界面总览
仅两个界面:**相机主界面**(默认全屏展示)+ **设置页**(右上角齿轮入口)。
### 5.2 相机主界面(核心界面)
```
┌─────────────────────────────────┐
│ [低光] [频率] [前/后摄] │ ← 顶部工具
│ 相机预览画面(全屏) │
│ ┌─────────────────────────────┐ │
│ │ ┌───────┐ │ │
│ │ │ 野鸡 86% │ │ │ ← 检测框(按类别着色)
│ │ │ 约 25m │ │ │
│ │ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ │ │ ← 疑似生境区域(黄色虚线,预警)
│ │ 疑似区域 · 约 20m │ │
│ │ └───────┘ │ │
│ └─────────────────────────────┘ │
│ [提醒开关] [设置] │ ← 底部操作
└─────────────────────────────────┘
```
交互说明:
- 顶部:检测频率切换(连续 / 标准 / 省电)、低光增强开关(P2)、前后摄像头切换;
- 中部:全屏预览,检测框 + 类别 + 置信度实时叠加;
- 底部:提醒开关(震动 / 声音)、设置入口;
- 支持双指缩放、点击对焦;
- 检测到目标时边框高亮 + 震动 / 声音提醒(10s 防重复);
- 检测框内显示类别、置信度与大致距离(约 X m),随目标实时更新;
- 未检测到目标时,疑似生境区域以黄色虚线框展示(含距离标注)并预警(提醒方式与动物检测区分)。
### 5.3 识别结果卡片(点击检测框弹出)
| 元素 | 说明 |
| --- | --- |
| 物种名称 | 中文名 + 英文标签(如:野鸡 pheasant |
| 置信度 | 百分比进度条 |
| 时间 | 检测时刻 |
| 距离 | 约 25m(单目估算,误差 ±30%) |
仅展示信息,无保存操作。
疑似生境区域(黄色虚线框)点击后展示:区域类别、置信度、距离(约 X m,按参考植被高度估算,误差较大)。
### 5.4 设置页
| 分组 | 设置项 | 默认值 |
| --- | --- | --- |
| 识别 | 置信度阈值 | 0.40 |
| 识别 | 检测频率 | 连续 |
| 识别 | 低光增强 | 关(P2) |
| 识别 | 距离标注 | 开 |
| 识别 | 生境区域阈值 | 0.35 |
| 提醒 | 震动提醒 / 提示音 / 防重复时长 | 开 / 开 / 10s |
| 提醒 | 生境区域预警 | 开 |
| 关于 | 版本信息 / 隐私说明 | — |
---
## 6. 权限设计
| 权限 | 用途 | 时机 | 备注 |
| --- | --- | --- | --- |
| CAMERA | 实时预览与识别 | 首次启动 | 必须 |
原则:**最小权限、按需申请**。仅申请相机权限;拒绝后提供说明引导页(可跳转系统设置重新授权)。不申请定位、存储等任何其他权限。
---
## 7. 非功能需求
| 类别 | 要求 |
| --- | --- |
| 兼容性 | Android 7.0+minSdk 24),竖屏为主,支持暗色模式 |
| 性能 | 见《技术方案》第 7 章(延迟 ≤ 80ms、启动 ≤ 1.5s、内存 ≤ 200MB |
| 离线 | 飞行模式下实时识别全流程完整可用 |
| 稳定性 | 崩溃率 ≤ 0.5%,启动成功率 ≥ 99%,异常自动降级(GPU 回退 CPU) |
| 耗电 | 连续使用 1 小时 ≤ 15%;省电模式 ≤ 8% |
| 隐私 | 无广告 SDK、无埋点、不保存任何数据,识别数据仅存内存 |
| 无障碍 | 关键操作支持 TalkBack 描述;检测结果支持语音播报(V2) |
---
## 8. 版本规划
| 版本 | 范围 | 说明 |
| --- | --- | --- |
| MVP / V1.0 | F01~F05、F07、F08 | 相机 + 实时识别 + 生境预警 + 提醒 + 设置,即完整核心体验 |
| V2.0 | +F06 及增强 | 低光增强、更多物种、夜视模式、模型远程更新、语音播报、距离精度增强(地面平面法) |
---
## 9. 合规与安全说明
- App 定位为**野生动物观察、识别工具**,不提供猎捕、诱捕、伤害野生动物的功能或指导;
- 首次启动展示合规提示:遵守《中华人民共和国野生动物保护法》,野鸡属"三有"保护动物,猎捕须依法许可;观察时保持距离、不惊扰动物、遵守保护区规定;
- 上架需提供隐私政策,声明仅使用相机权限、不采集与存储任何用户数据。
+512
View File
@@ -0,0 +1,512 @@
# Observer(野视)· 技术实现文档
| 项目 | 内容 |
| --- | --- |
| 文档版本 | v1.1 |
| 编写日期 | 2026-08-17 |
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照、记录、相册识别、定位相关实现) |
| 配套文档 | 《01-技术方案》《02-项目功能文档》 |
---
## 1. 项目结构
```
app/
├── src/main/
│ ├── java/com/example/observer/
│ │ ├── MainActivity.kt // 单 Activity 入口
│ │ ├── camera/
│ │ │ ├── CameraController.kt // CameraX 生命周期绑定与用例组合
│ │ │ └── FrameAnalyzer.kt // ImageAnalysis 帧分析器(节流 + 调度)
│ │ ├── detection/
│ │ │ ├── Detector.kt // 检测器抽象接口
│ │ │ ├── TFLiteDetector.kt // TFLite 实现(YOLOv8n
│ │ │ ├── DetectionResult.kt // 检测结果数据类
│ │ │ ├── Nms.kt // NMS 后处理
│ │ │ └── CoordinateMapper.kt // 模型坐标 → 视图坐标映射
│ │ ├── distance/
│ │ │ └── DistanceEstimator.kt // 单目距离估计(针孔模型)
│ │ ├── overlay/
│ │ │ └── DetectionOverlay.kt // Compose 叠加层(Canvas 绘制检测框)
│ │ ├── reminder/
│ │ │ └── Reminder.kt // 震动 / 提示音提醒(含防重复)
│ │ ├── ui/
│ │ │ ├── camera/ CameraScreen.kt / CameraViewModel.kt
│ │ │ └── settings/ SettingsScreen.kt / SettingsViewModel.kt
│ │ └── util/
│ │ └── BitmapUtils.kt // 缩放 / 旋转 / 复用
│ ├── assets/
│ │ ├── model.tflite // YOLOv8n 四类模型
│ │ └── labels.txt // 类别标签(按 ID 顺序)
│ └── res/
└── build.gradle.kts
```
---
## 2. 技术栈与版本
| 组件 | 版本(以最新稳定为准) | 用途 |
| --- | --- | --- |
| Kotlin | 2.x | 开发语言 |
| AGP | 8.x | Android Gradle 插件 |
| Jetpack Compose | BOM 2024.x+Material 3 | UI |
| CameraX | 1.4.x | Preview / ImageAnalysis |
| TensorFlow Lite | 2.16.x | 端侧推理 |
| tensorflow-lite-gpu | 2.16.x | GPU 加速 |
| DataStore | 1.1.x | 设置存储 |
依赖(build.gradle.kts 关键片段):
```kotlin
dependencies {
implementation("androidx.camera:camera-core:1.4.1")
implementation("androidx.camera:camera-camera2:1.4.1")
implementation("androidx.camera:camera-lifecycle:1.4.1")
implementation("androidx.camera:camera-view:1.4.1")
implementation("org.tensorflow:tensorflow-lite:2.16.1")
implementation("org.tensorflow:tensorflow-lite-gpu:2.16.1")
implementation("androidx.datastore:datastore-preferences:1.1.1")
}
```
---
## 3. 模块设计
### 3.1 camera 模块 — CameraController.kt
职责:创建并绑定 CameraX 用例(Preview + ImageAnalysis),统一生命周期。
```kotlin
class CameraController(
private val lifecycleOwner: LifecycleOwner,
private val analysisExecutor: Executor,
) {
private lateinit var cameraProvider: ProcessCameraProvider
private lateinit var imageAnalysis: ImageAnalysis
fun start(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer) {
val future = ProcessCameraProvider.getInstance(previewView.context)
future.addListener({
cameraProvider = future.get()
val preview = Preview.Builder().build().also {
it.surfaceProvider = previewView.surfaceProvider
}
imageAnalysis = ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build()
imageAnalysis.setAnalyzer(analysisExecutor, analyzer)
cameraProvider.bindToLifecycle(
lifecycleOwner,
CameraSelector.DEFAULT_BACK_CAMERA,
preview, imageAnalysis,
)
}, ContextCompat.getMainExecutor(previewView.context))
}
}
```
要点:
- `OUTPUT_IMAGE_FORMAT_RGBA_8888`CameraX 1.3+)直接得到 RGBA 图像,免 YUV 转换;
- `STRATEGY_KEEP_ONLY_LATEST`:分析器忙时丢弃旧帧,不积压;
- 相机权限检查与请求在进入本模块前完成;
- 前后摄切换:重新以对应 `CameraSelector` 执行 `bindToLifecycle`(解绑旧用例)。
### 3.2 camera 模块 — FrameAnalyzer.kt
职责:帧节流、旋转获取、调度检测、结果回调。
```kotlin
class FrameAnalyzer(
private val detector: Detector,
private val onResult: (List<DetectionResult>, Int) -> Unit, // 结果 + 旋转角
) : ImageAnalysis.Analyzer {
private var lastDetectMs = 0L
override fun analyze(imageProxy: ImageProxy) {
val now = SystemClock.elapsedRealtime()
val interval = frameIntervalMs() // 依据设置:连续/标准/省电
if (now - lastDetectMs < interval) {
imageProxy.close(); return
}
lastDetectMs = now
val bitmap = imageProxy.toBitmap() // RGBA_8888 直接转 Bitmap
val rotation = imageProxy.imageInfo.rotationDegrees
try {
val results = detector.detect(bitmap)
onResult(results, rotation)
} finally {
imageProxy.close() // 必须关闭,否则阻塞流
}
}
}
```
### 3.3 detection 模块 — Detector 接口
```kotlin
interface Detector {
/** 输入 RGBA 位图,输出归一化坐标检测结果(0..1) */
fun detect(bitmap: Bitmap): List<DetectionResult>
}
```
### 3.4 detection 模块 — TFLiteDetector.kt
```kotlin
class TFLiteDetector(
context: Context,
private val inputSize: Int = 320,
private val confThreshold: Float = 0.40f, // 动物类阈值
private val habitatThreshold: Float = 0.35f, // 生境区域阈值
private val iouThreshold: Float = 0.45f,
gpuEnabled: Boolean = true,
) : Detector {
private val labels: List<String> =
context.assets.open("labels.txt").bufferedReader().readLines()
private val interpreter: Interpreter = Interpreter(
loadModelFile(context, "model.tflite"),
Interpreter.Options().apply {
setNumThreads(4)
if (gpuEnabled) {
try { addDelegate(GpuDelegate()) } catch (_: Exception) { /* 回退 CPU */ }
}
},
)
private val inputBuffer: ByteBuffer = ByteBuffer.allocateDirect(
1 * inputSize * inputSize * 3 * 4 // float32
).order(ByteOrder.nativeOrder())
private val outputBuffer: ByteBuffer = ByteBuffer.allocateDirect(
OUTPUT_ELEMENTS * 4
).order(ByteOrder.nativeOrder())
override fun detect(bitmap: Bitmap): List<DetectionResult> {
preprocess(bitmap, inputBuffer) // 缩放 + RGB 归一化 → 输入缓冲
interpreter.run(inputBuffer, outputBuffer) // 单张推理
return postprocess(outputBuffer, bitmap.width, bitmap.height)
}
}
```
输入 / 输出规格(YOLOv8n 四类,320 输入):
- 输入:`[1, 320, 320, 3]`RGBfloat32 归一化 0~1
- 输出:`[1, 9, 8400]`8400 = 各尺度 anchor 数,9 = 4 个框坐标 cx/cy/w/h + 5 类得分),展平为 `9 * 8400` 个 float。
### 3.5 detection 模块 — 后处理(解码 + NMS)
```kotlin
private fun postprocess(raw: FloatArray, imgW: Int, imgH: Int): List<DetectionResult> {
val numAnchor = raw.size / 9
val boxes = mutableListOf<DetectionResult>()
for (a in 0 until numAnchor) {
val cx = raw[a]; val cy = raw[9 + a]
val w = raw[18 + a]; val h = raw[27 + a]
var bestCls = 0; var bestScore = 0f
for (c in 0 until 5) {
val s = raw[36 + c * numAnchor + a] // 类别得分按 anchor 平铺
if (s > bestScore) { bestScore = s; bestCls = c }
}
if (bestScore < confThreshold) continue
boxes += DetectionResult(
label = labels[bestCls],
score = bestScore,
// 归一化坐标(裁剪到 [0,1])
left = (cx - w / 2).coerceIn(0f, 1f),
top = (cy - h / 2).coerceIn(0f, 1f),
right = (cx + w / 2).coerceIn(0f, 1f),
bottom = (cy + h / 2).coerceIn(0f, 1f),
)
}
return nms(boxes, iouThreshold)
}
fun nms(boxes: List<DetectionResult>, iouThreshold: Float): List<DetectionResult> {
val sorted = boxes.sortedByDescending { it.score }
val kept = mutableListOf<DetectionResult>()
for (b in sorted) {
if (kept.none { iou(b, it) > iouThreshold }) kept += b
}
return kept
}
```
注意:以上取数索引为示意,**必须与模型导出时的输出布局(yolov8 tflite 为 9×8400,按列平铺)核对一致**,建议训练导出后用 Python 脚本先对单图做一致性校验再接入 App。
### 3.6 distance 模块 — DistanceEstimator.kt
```kotlin
class DistanceEstimator(context: Context) {
private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
private var focalPxCache = -1f
// 物种参考体型(米),用于针孔模型估算
private val speciesSizeM = mapOf(
"pheasant" to 0.45f, // 身高
)
/** 距离 = 焦距px × 参考体型 / 框高px(在分析图像分辨率下计算) */
fun estimate(label: String, boxHeightNorm: Float, imageHeightPx: Int): Float? {
val realH = speciesSizeM[label] ?: return null
val boxH = boxHeightNorm * imageHeightPx
if (boxH < 8f) return null // 过小目标不估算
val focalPx = focalPx(imageHeightPx)
if (focalPx <= 0) return null
return round(focalPx * realH / boxH)
}
/** focal_px = focal_mm × (imageHeightPx / sensorHeightMm);视场角作回退 */
private fun focalPx(imageHeightPx: Int): Float {
if (focalPxCache > 0) return focalPxCache
val c = cameraManager.getCameraCharacteristics(cameraManager.cameraIdList.first())
val focalMm = c.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)?.firstOrNull()
val sensor = c.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)
val fov = c.get(CameraCharacteristics.LENS_INFO_HORIZONTAL_VIEW_ANGLE)
focalPxCache = when {
focalMm != null && sensor != null -> focalMm * imageHeightPx / sensor.height
fov != null -> (imageHeightPx / 2f) / tan(fov / 2f)
else -> -1f
}
return focalPxCache
}
}
```
说明:
- 前后摄切换时需按当前相机重新获取焦距(缓存按相机 ID 区分);
- 距离估算仅供参考,实际距离可能因环境因素有所偏差。
### 3.7 detection 模块 — CoordinateMapper.kt
模型输出为归一化坐标(相对分析图像,竖屏方向),需转换到预览视图坐标:
```kotlin
class CoordinateMapper(private val view: PreviewView) {
/** 归一化坐标 → 预览视图像素坐标(考虑传感器旋转与 FIT_CENTER 裁剪) */
fun mapToView(norm: DetectionResult, rotation: Int): RectF {
// 1) 旋转校正:把"图像方向"归一化坐标转到"竖屏视图方向"
val (x0, y0, x1, y1) = rotate(norm, rotation)
// 2) 处理 FIT_CENTER 的裁剪偏移与缩放
val viewW = view.width; val viewH = view.height
val scale = min(viewW / bitmapW, viewH / bitmapH) // bitmap 尺寸
val offsetX = (viewW - bitmapW * scale) / 2f
val offsetY = (viewH - bitmapH * scale) / 2f
return RectF(
x0 * bitmapW * scale + offsetX,
y0 * bitmapH * scale + offsetY,
x1 * bitmapW * scale + offsetX,
y1 * bitmapH * scale + offsetY,
)
}
}
```
### 3.8 overlay 模块 — DetectionOverlay.kt
Compose Canvas 叠加层:
```kotlin
@Composable
fun DetectionOverlay(results: List<DetectionResult>, rotation: Int, modifier: Modifier) {
val mapper = remember { CoordinateMapper(view) }
Canvas(modifier = modifier) {
results.forEach { r ->
val rect = mapper.mapToView(r, rotation)
drawRect(
color = colorOf(r.label),
topLeft = Offset(rect.left, rect.top),
size = Size(rect.width(), rect.height()),
style = Stroke(6.dp.toPx()),
)
// 距离由 ViewModel 计算后写入 DetectionResult.distanceFloat?
val dist = r.distance?.let { " · 约${it}m" } ?: ""
drawText("${r.label} ${(r.score * 100).toInt()}%$dist")
}
}
}
```
防闪烁:ViewModel 中对结果做"目标停留 ≥ 0.5s 才显示、消失 2s 后移除"的平滑处理。
### 3.9 reminder 模块 — Reminder.kt
```kotlin
class Reminder(private val context: Context) {
private val vibrator = context.getSystemService(Vibrator::class.java)
private var lastAlertAt = 0L
private var lastAlertLabel: String? = null
/** 同类目标 10s 内只提醒一次 */
fun onDetected(label: String) {
val now = SystemClock.elapsedRealtime()
if (label == lastAlertLabel && now - lastAlertAt < 10_000) return
lastAlertAt = now
lastAlertLabel = label
vibrator?.vibrate(VibrationEffect.createOneShot(200, VibrationEffect.DEFAULT_AMPLITUDE))
}
}
```
### 3.10 设置存储 — DataStore
使用 DataStore Preferences 保存:`conf_threshold`(默认 0.40)、`detect_mode`(连续/标准/省电)、`vibrate_enabled``sound_enabled``low_light_enhance`P2)、`show_distance`(默认开)。`SettingsViewModel` 以 Flow 暴露,`FrameAnalyzer``TFLiteDetector` 读取最新值。
---
## 4. 关键流程实现
### 4.1 启动流程
```
MainActivity.onCreate
→ 检查/请求 CAMERA 权限
→ 初始化 TFLiteDetector(后台线程,含预热推理)
→ CameraController.start(previewView, analyzer)
→ ViewModel 订阅检测结果 → 叠加层渲染 + 提醒触发
```
预热:加载模型后执行一次空推理(全零输入),避免首帧卡顿。
### 4.2 帧节流与并发控制
- 检测间隔:连续 80~100ms / 标准 300ms / 省电 1000ms(由设置决定);
- 推理在单线程 ExecutorHandlerThread)串行执行,天然互斥;
- 分析线程只做"取帧 → 判断节流 → 提交任务",不做推理,保证预览流畅。
### 4.3 提醒、距离标注与防闪烁
- 检测到目标(置信度 ≥ 阈值,生境区域用独立阈值)→ `Reminder.onDetected(label)`,10s 防重复,动物与生境预警提醒方式区分;
- 叠加层展示逻辑:目标连续出现 ≥ 0.5s 才绘制;消失后 2s 内不清除,避免闪烁;
- 低置信度结果(阈值以下)不展示、不提醒;
- 距离标注:结果到达后调用 `DistanceEstimator` 估算"约 X m",随检测框渲染(FrameAnalyzer 回调携带分析图像高度);生境区域同样估算(按参考植被高度,误差较大);框高过小(< 8px)或无法获取焦距显示"--"。
---
## 5. 模型集成细节
| 项 | 说明 |
| --- | --- |
| 模型文件 | `app/src/main/assets/model.tflite`,随 APK 打包 |
| 标签文件 | `assets/labels.txt`,每行一个类别,顺序与训练一致(pheasant) |
| 量化 | 优先 fp16GPU);int8 用于 CPU/低端机回退 |
| 加载 | `Interpreter(loadAssetFile(...))`,进程内单例 |
| 预热 | 初始化后执行一次 dummy run |
| 校验 | 发布前用 Python 脚本(tflite-runtime)对测试集抽样验证模型输出布局与 App 解析一致 |
---
## 6. 性能优化实践
| 手段 | 说明 |
| --- | --- |
| GPU Delegate | 优先启用;`addDelegate` 抛异常或首帧异常时回退 CPU,并上报降级日志 |
| 输入缓冲复用 | ByteBuffer 复用,避免每次检测重新分配 |
| 帧节流 | 按模式降频,KEEP_ONLY_LATEST 不积压 |
| 位图复用 | ImageProxy → Bitmap 走共享缓冲;叠加层仅绘制检测框,不复制整帧 |
| 线程模型 | 检测线程单线程串行;UI 线程零推理 |
| 低分辨率检测 | 320 输入检测;远距离模式可切 416(V2) |
| 距离标注 | 纯算术运算开销可忽略;焦距 px 按相机缓存复用,避免重复查询 CameraCharacteristics |
| 生命周期 | 后台自动 `unbind` 相机释放资源,避免耗电 |
| 省电模式 | 降到 1 帧/秒 + 低分辨率 + 关闭提醒以外的动画 |
---
## 7. 异常与降级策略
| 异常场景 | 处理 |
| --- | --- |
| GPU 不可用 / 模型加载失败 | 回退 CPU 线程数调整;失败则提示"识别不可用"但不影响相机预览 |
| 相机初始化失败(个别机型) | 重试一次 → 失败提示并引导检查权限/重启 |
| 权限被拒 | 引导页说明用途,提供跳转系统设置 |
| 低内存 | 主动释放缓存位图,降低检测分辨率 |
| 推理超时(> 500ms) | 丢弃该帧结果,恢复下一帧,保证预览流畅 |
| 无法读取焦距 / 视场角(个别机型) | 距离显示"--",识别不受影响 |
---
## 8. 测试方案
### 8.1 单元测试(JUnit
- `NmsTest`:NMS 正确性(重叠/多类别/边界框);
- `CoordinateMapperTest`:四向旋转映射、FIT_CENTER 裁剪偏移;
- `ThresholdTest`:置信度阈值过滤逻辑;
- `ReminderTest`:10s 防重复提醒逻辑、动物与生境提醒方式区分;
- `DistanceEstimatorTest`:焦距换算、距离公式、过小目标不估算、生境区域按植被高度估算。
### 8.2 模型评估(Python 脚本,独立于 App)
- 对测试集计算 mAP@0.5、mAP@0.5:0.95、各类别 AP、负样本误检率;
- 输出混淆矩阵,分析检测精度。
### 8.3 仪器测试(androidTest
- 模拟帧注入:将测试图片经 `ImageProxy` 注入分析器,断言结果(不依赖真机取景);
- 权限拒绝 / 恢复场景;
- 冷启动到预览的时间基准测试。
### 8.4 真机测试矩阵
| 机型档位 | 代表机型 | 验证项 |
| --- | --- | --- |
| 旗舰 | 骁龙 8 系 / 麒麟 9 系 | 全功能、GPU 路径、长时间发热 |
| 中端 | 骁龙 7 系 / 天玑 8 系 | 延迟 ≤ 80ms、耗电 ≤ 15%/h |
| 低端 | 骁龙 4 系 / 天玑 6 系 | CPU 回退路径、省电模式可用性 |
---
## 9. 构建与发布
```kotlin
android {
compileSdk = 35
defaultConfig {
applicationId = "com.example.observer"
minSdk = 24
targetSdk = 35
ndk { abiFilters += listOf("arm64-v8a") } // 主发 arm64;如需兼容 32 位另发包
}
buildTypes {
release {
isMinifyEnabled = true
isShrinkResources = true
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
}
}
}
```
- R8 规则:保留 TFLite 相关类(`-keep class org.tensorflow.** { *; }`),assets 模型不混淆;
- 签名:正式签名 + Gradle 管理(或环境变量注入),不上传 keystore 到仓库;
- 上架检查:隐私政策(仅相机权限、不采集与存储数据声明)、权限用途说明。
---
## 10. 后续演进(V2 方向)
- 更多物种类别(哺乳类、鸟类细分),支持模型远程更新;
- 小目标优化:SAHI 切图推理 / 专用小目标模型 / 数字变焦辅助;
- 夜视 / 红外增强模式;
- 检测结果语音播报,提升无障碍体验;
- 距离精度增强:基于设备俯仰角与相机高度的地面平面法、镜头畸变校正、水下折射修正;
- 生境类别细分与精度提升(湿地、林缘等),生境预警策略优化;
- 若未来需要留存能力(拍照、记录),可基于现有检测链路平滑扩展。
+4
View File
@@ -0,0 +1,4 @@
org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
android.useAndroidX=true
kotlin.code.style=official
android.nonTransitiveRClass=true
+39
View File
@@ -0,0 +1,39 @@
[versions]
agp = "8.5.2"
kotlin = "2.0.20"
coreKtx = "1.13.1"
lifecycle = "2.8.6"
activityCompose = "1.9.2"
composeBom = "2024.09.03"
camerax = "1.4.1"
tflite = "2.16.1"
datastore = "1.1.1"
coroutines = "1.9.0"
junit = "4.13.2"
[libraries]
androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" }
androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" }
androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" }
androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" }
androidx-compose-ui = { group = "androidx.compose.ui", name = "ui" }
androidx-compose-ui-graphics = { group = "androidx.compose.ui", name = "ui-graphics" }
androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling" }
androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview" }
androidx-compose-material3 = { group = "androidx.compose.material3", name = "material3" }
androidx-camera-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" }
androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" }
androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" }
androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" }
org-tensorflow-lite = { group = "org.tensorflow", name = "tensorflow-lite", version.ref = "tflite" }
org-tensorflow-lite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tflite" }
androidx-datastore-preferences = { group = "androidx.datastore", name = "datastore-preferences", version.ref = "datastore" }
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" }
junit = { group = "junit", name = "junit", version.ref = "junit" }
[plugins]
android-application = { id = "com.android.application", version.ref = "agp" }
kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }
Binary file not shown.
+9
View File
@@ -0,0 +1,9 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
networkTimeout=10000
retries=0
retryBackOffMs=500
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
Vendored Executable
+248
View File
@@ -0,0 +1,248 @@
#!/bin/sh
#
# Copyright © 2015 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#
##############################################################################
#
# gradlew start up script for POSIX generated by Gradle.
#
# Important for running:
#
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
# noncompliant, but you have some other compliant shell such as ksh or
# bash, then to run this script, type that shell name before the whole
# command line, like:
#
# ksh gradlew
#
# Busybox and similar reduced shells will NOT work, because this script
# requires all of these POSIX shell features:
# * functions;
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
# * compound commands having a testable exit status, especially «case»;
# * various built-in commands including «command», «set», and «ulimit».
#
# Important for patching:
#
# (2) This script targets any POSIX shell, so it avoids extensions provided
# by Bash, Ksh, etc; in particular arrays are avoided.
#
# The "traditional" practice of packing multiple parameters into a
# space-separated string is a well documented source of bugs and security
# problems, so this is (mostly) avoided, by progressively accumulating
# options in "$@", and eventually passing that to Java.
#
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
# see the in-line comments for details.
#
# There are tweaks for specific operating systems such as AIX, CygWin,
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
#
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
app_path=$0
# Need this for daisy-chained symlinks.
while
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
[ -h "$app_path" ]
do
ls=$( ls -ld "$app_path" )
link=${ls#*' -> '}
case $link in #(
/*) app_path=$link ;; #(
*) app_path=$APP_HOME$link ;;
esac
done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
warn () {
echo "$*"
} >&2
die () {
echo
echo "$*"
echo
exit 1
} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "$( uname )" in #(
CYGWIN* ) cygwin=true ;; #(
Darwin* ) darwin=true ;; #(
MSYS* | MINGW* ) msys=true ;; #(
NONSTOP* ) nonstop=true ;;
esac
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD=$JAVA_HOME/jre/sh/java
else
JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD=java
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
case $MAX_FD in #(
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
fi
# Collect all arguments for the java command, stacking in reverse order:
# * args from the command line
# * the main class name
# * -classpath
# * -D...appname settings
# * --module-path (only if needed)
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
# Now convert the arguments - kludge to limit ourselves to /bin/sh
for arg do
if
case $arg in #(
-*) false ;; # don't mess with options #(
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
[ -e "$t" ] ;; #(
*) false ;;
esac
then
arg=$( cygpath --path --ignore --mixed "$arg" )
fi
# Roll the args list around exactly as many times as the number of
# args, so each arg winds up back in the position where it started, but
# possibly modified.
#
# NB: a `for` loop captures its iteration list before it begins, so
# changing the positional parameters here affects neither the number of
# iterations, nor the values presented in `arg`.
shift # remove old arg
set -- "$@" "$arg" # push replacement arg
done
fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"
# Stop when "xargs" is not available.
if ! command -v xargs >/dev/null 2>&1
then
die "xargs is not available"
fi
# Use "xargs" to parse quoted args.
#
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
#
# In Bash we could simply go:
#
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
# set -- "${ARGS[@]}" "$@"
#
# but POSIX shell has neither arrays nor command substitution, so instead we
# post-process each arg (as a line of input to sed) to backslash-escape any
# character that might be a shell metacharacter, then use eval to reverse
# that process (while maintaining the separation between arguments), and wrap
# the whole thing up as a single "set" statement.
#
# This will of course break if any of these variables contains a newline or
# an unmatched quote.
#
eval "set -- $(
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
xargs -n1 |
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
tr '\n' ' '
)" '"$@"'
exec "$JAVACMD" "$@"
Vendored
+82
View File
@@ -0,0 +1,82 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem
@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
@rem
@rem gradlew startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables, and ensure extensions are enabled
setlocal EnableExtensions
set DIRNAME=%~dp0
if "%DIRNAME%"=="" set DIRNAME=.
@rem This is normally unused
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:execute
@rem Setup the command line
@rem Execute gradlew
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
@rem which allows us to clear the local environment before executing the java command
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
:exitWithErrorLevel
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
"%COMSPEC%" /c exit %ERRORLEVEL%
+24
View File
@@ -0,0 +1,24 @@
pluginManagement {
repositories {
google {
content {
includeGroupByRegex("com\\.android.*")
includeGroupByRegex("com\\.google.*")
includeGroupByRegex("androidx.*")
}
}
mavenCentral()
gradlePluginPortal()
}
}
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
repositories {
google()
mavenCentral()
}
}
rootProject.name = "Observer"
include(":app")
+202
View File
@@ -0,0 +1,202 @@
"""用 LocalAI qwen3.5-9b 多模态模型自动标注图片。
用法:python auto_label.py
输入:datasets/images/<class>/*.jpg
输出:datasets/dataset/{train,val}/{images,labels} (YOLO 格式) + data.yaml
"""
import os
import json
import re
import base64
import shutil
import sys
import urllib.request
from PIL import Image, ImageDraw
# LocalAI 服务器配置
LOCAL_AI_URL = "http://192.168.3.210:18080/v1/chat/completions"
MODEL_NAME = "qwen3.5-9b"
# 标注配置
KEEP_PER_CLASS = 100
MIN_CONF = 0.5 # 自动标注最低置信度
MIN_BOX = 0.03 # 框面积占比下限
BASE = os.path.dirname(__file__)
IMG_DIR = os.path.join(BASE, "datasets", "images")
OUT_DIR = os.path.join(BASE, "datasets", "dataset")
PREVIEW_DIR = os.path.join(BASE, "datasets", "preview")
CLASSES = ["pheasant"]
PROMPTS = {
"pheasant": "pheasant",
}
# 标注提示词模板
ANNOTATION_PROMPT = """你是一个目标检测助手。请检测图片中【{target}】动物。
输出格式(严格只输出这一行,不要其他文字):
pheasant 0.92 0.1 0.2 0.3 0.4
字段说明:
- 第一个数字:置信度(0-1
- 后四个数字:边界框坐标 x1 y1 x2 y2(归一化 0-1
如果没找到目标,只输出:NOT_FOUND
"""
def encode_image_to_base64(path: str) -> str:
"""将图片路径转换为 base64 数据"""
with open(path, "rb") as f:
return base64.b64encode(f.read()).decode("utf-8")
def call_localai(image_path: str, prompt: str) -> dict:
"""调用 LocalAI qwen3.5-9b 进行标注"""
image_b64 = encode_image_to_base64(image_path)
data = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
],
}
],
"temperature": 0.1,
}
try:
with urllib.request.urlopen(LOCAL_AI_URL, data=json.dumps(data).encode("utf-8")) as resp:
if resp.status == 200:
result = json.loads(resp.read().decode("utf-8"))
return {"success": True, "data": result}
else:
return {"success": False, "error": f"HTTP {resp.status}"}
except Exception as e:
return {"success": False, "error": str(e)}
def parse_annotation(response: dict) -> list:
"""解析 LocalAI qwen3.5-9b 返回的标注结果(文本格式)"""
if not response.get("success"):
return []
content = response.get("choices", [{}])[0].get("message", {}).get("content", "")
# 如果没找到目标
if "NOT_FOUND" in content or "未找到" in content:
return []
# 解析文本格式:类别 置信度 x1 y1 x2 y2
# 例如:pheasant 0.92 0.1 0.2 0.3 0.4
parts = content.strip().split()
if len(parts) < 5:
return []
cls_name = parts[0].lower()
try:
confidence = float(parts[1])
x1, y1, x2, y2 = [float(v) for v in parts[2:6]]
except ValueError:
return []
# 确保坐标顺序
if x1 > x2 or y1 > y2:
x1, x2 = x2, x1
y1, y2 = y2, y1
return [{
"class": cls_name,
"confidence": confidence,
"bbox": [x1, y1, x2, y2],
}]
def main() -> None:
for split in ("train", "val"):
for sub in ("images", "labels"):
os.makedirs(os.path.join(OUT_DIR, split, sub), exist_ok=True)
os.makedirs(PREVIEW_DIR, exist_ok=True)
for cls in CLASSES:
src_dir = os.path.join(IMG_DIR, cls)
files = sorted(f for f in os.listdir(src_dir) if f.endswith(".jpg"))
print(f"[label] {cls}: {len(files)}", flush=True)
kept = []
for f in files:
path = os.path.join(src_dir, f)
image = Image.open(path).convert("RGB")
w, h = image.size
# 调用 LocalAI 进行标注
prompt = ANNOTATION_PROMPT.format(target=cls)
result = call_localai(path, prompt)
annotations = parse_annotation(result)
# 取置信度最高的标注
best = None
best_score = 0
for ann in annotations:
if ann.get("class") != cls:
continue
score = ann.get("confidence", 0)
if score > best_score:
best_score = score
best = ann
if best is None:
print(f" {f}: ✗ (未找到目标)", flush=True)
continue
if best_score < MIN_CONF:
print(f" {f}: ✗ (置信度 {best_score:.2f} < {MIN_CONF})", flush=True)
continue
x1, y1, x2, y2 = best.get("bbox", [0, 0, 1, 1])
kept.append((path, (best_score, x1, y1, x2, y2)))
print(f" {f}: ✓ ({best_score:.2f})", flush=True)
kept.sort(key=lambda t: t[1][0], reverse=True)
kept = kept[:KEEP_PER_CLASS]
avg = sum(k[1][0] for k in kept) / max(len(kept), 1)
print(f" → 保留 {len(kept)} 张(平均置信度 {avg:.2f}", flush=True)
for i, (path, (score, x1, y1, x2, y2)) in enumerate(kept):
split = "train" if i % 10 else "val"
dst_img = os.path.join(OUT_DIR, split, "images", f"{cls}_{i:03d}.jpg")
shutil.copy(path, dst_img)
# 计算 YOLO 格式坐标
w, h = Image.open(dst_img).size
cx = (x1 + x2) / 2 / w
cy = (y1 + y2) / 2 / h
bw = (x2 - x1) / w
bh = (y2 - y1) / h
label_file = os.path.join(OUT_DIR, split, "labels", f"{cls}_{i:03d}.txt")
with open(label_file, "w") as f:
f.write(f"{CLASSES.index(cls)} {cx:.6f} {cy:.6f} {bw:.6f} {bh:.6f}\n")
# 生成预览图
img = Image.open(dst_img).convert("RGB")
draw = ImageDraw.Draw(img)
draw.rectangle([x1, y1, x2, y2], outline="#E53935", width=3)
draw.text((x1, max(y1 - 14, 0)), f"{cls} {score:.0%}", fill="#E53935")
img.save(os.path.join(PREVIEW_DIR, f"{cls}_{i:03d}.jpg"), "JPEG", quality=85)
# 生成 data.yaml
with open(os.path.join(OUT_DIR, "data.yaml"), "w") as f:
f.write(f"path: {OUT_DIR}\n")
f.write("train: train/images\nval: val/images\n")
f.write(f"names: {CLASSES}\n")
print("[done] 数据集就绪:", OUT_DIR, flush=True)
if __name__ == "__main__":
sys.exit(main())
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""
为训练图片自动生成标注
使用 YOLO 模型检测野鸡并生成 YOLO 格式标注
"""
import cv2
import os
from pathlib import Path
from ultralytics import YOLO
# 配置
MODEL_PATH = Path(__file__).parent / "yolov8s-world.pt"
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
def create_yolo_structure():
"""创建 YOLO 格式目录结构"""
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
def convert_to_yolo_format(bbox, img_width, img_height):
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
x1, y1, x2, y2 = bbox
cx = (x1 + x2) / 2 / img_width
cy = (y1 + y2) / 2 / img_height
w = (x2 - x1) / img_width
h = (y2 - y1) / img_height
return cx, cy, w, h
def main():
# 加载模型
print(f"加载模型: {MODEL_PATH}")
model = YOLO(str(MODEL_PATH))
# 创建目录结构
create_yolo_structure()
# 获取所有图片
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
image_files = [
f for f in IMAGES_DIR.iterdir()
if f.suffix.lower() in image_extensions
]
print(f"找到 {len(image_files)} 张图片")
# 处理每张图片
labeled_count = 0
for img_path in image_files:
print(f"处理: {img_path.name}")
# 读取图片
image = cv2.imread(str(img_path))
if image is None:
print(f" 跳过: 无法读取 {img_path.name}")
continue
img_height, img_width = image.shape[:2]
# 推理
results = model(image, conf=0.3, iou=0.45)
# 收集标注
labels = []
for result in results:
boxes = result.boxes
if boxes is None or len(boxes) == 0:
continue
for box in boxes:
class_name = result.names.get(int(box.cls[0]), "")
# 只保留 bird 或 pheasant 类别
if class_name in ["bird", "pheasant"]:
class_id = 0 # 只有一个类别:野鸡
bbox = list(map(int, box.xyxy[0].tolist()))
conf = float(box.conf[0])
# 转换为 YOLO 格式
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
labels.append(f"{class_id} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
# 保存标注文件
if len(labels) > 0:
labeled_count += 1
# 复制图片
dst_img_path = OUTPUT_DIR / "images" / "train" / img_path.name
cv2.imwrite(str(dst_img_path), image)
# 保存标注
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
with open(label_path, "w") as f:
f.write("\n".join(labels))
print(f" ✓ 标注了 {len(labels)} 个目标")
else:
print(f" - 未检测到野鸡")
print(f"\n完成!")
print(f" 总图片数: {len(image_files)}")
print(f" 有效标注: {labeled_count}")
print(f" 输出目录: {OUTPUT_DIR}")
# 创建 data.yaml
data_yaml = OUTPUT_DIR / "data.yaml"
with open(data_yaml, "w") as f:
f.write(f"""# Observer 数据集配置 - 只识别野鸡
path: {OUTPUT_DIR}
train: images/train
val: images/train
# 类别
nc: 1
names: ['pheasant']
""")
print(f" 数据配置: {data_yaml}")
if __name__ == "__main__":
main()
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
"""野鸡图片全自动标注 v2。
方案:Grounding DINO(开放词汇检测,多提示词)→ NMS 去重 →
CLIP 裁剪验证(剔除误检)→ 仍无框时 CLIP 滑动窗口兜底。
输出 YOLO 格式标注 + 可视化预览,覆盖 datasets/images/pheasant 下全部图片。
"""
import os
import sys
import torch
import numpy as np
from pathlib import Path
from PIL import Image, ImageDraw
from torchvision.ops import nms
from transformers import (
GroundingDinoProcessor,
GroundingDinoForObjectDetection,
CLIPProcessor,
CLIPModel,
)
BASE = Path(__file__).parent
IMG_DIR = BASE / "datasets" / "images" / "pheasant"
OUT_DIR = BASE / "datasets" / "yolo_format"
PREVIEW_DIR = BASE / "datasets" / "preview"
DINO_PROMPTS = [
"pheasant",
"ring-necked pheasant",
"wild pheasant",
"common pheasant",
"bird",
]
DINO_BOX_THRESHOLD = 0.13
DINO_TEXT_THRESHOLD = 0.15
NMS_IOU = 0.45
KEEP_THRESHOLD = 0.16 # 低于此分的框必须通过 CLIP 验证才保留
MIN_BOX_AREA = 0.002 # 框面积占比下限(排除过小误检)
CLIP_POSITIVE = ["a photo of a pheasant", "a photo of a wild bird"]
CLIP_NEGATIVE = [
"a photo of grass and leaves",
"a photo of rocks and soil",
"a photo of a landscape",
"a photo of a fence",
"a photo of trees",
]
CLIP_VERIFY_THRESHOLD = 0.40 # softmax(正例) >= 此值才算验证通过
FALLBACK_WINDOW_SIZES = [0.5, 0.35, 0.25] # 滑动窗口占图宽比例
class AutoLabeler:
def __init__(self):
print("加载 Grounding DINO...", flush=True)
self.proc = GroundingDinoProcessor.from_pretrained("IDEA-Research/grounding-dino-tiny")
self.dino = GroundingDinoForObjectDetection.from_pretrained("IDEA-Research/grounding-dino-tiny")
self.dino.eval()
print("加载 CLIP...", flush=True)
self.clip_proc = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
self.clip = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
self.clip.eval()
pos_tokens = self.clip_proc(text=CLIP_POSITIVE + CLIP_NEGATIVE, return_tensors="pt", padding=True)
with torch.no_grad():
out = self.clip.get_text_features(**pos_tokens)
if hasattr(out, "text_embeds"):
feats = out.text_embeds
elif hasattr(out, "pooler_output"):
feats = out.pooler_output
else:
feats = out
feats = feats / feats.norm(dim=-1, keepdim=True)
self.pos_feats = feats[: len(CLIP_POSITIVE)]
self.neg_feats = feats[len(CLIP_POSITIVE):]
def clip_score(self, region: Image.Image) -> float:
"""返回裁剪区域是野鸡的概率(softmax 归一化,0~1)"""
inputs = self.clip_proc(images=region, return_tensors="pt")
with torch.no_grad():
out = self.clip.get_image_features(**inputs)
if hasattr(out, "pooler_output"):
img_feat = out.pooler_output
else:
img_feat = out
img_feat = img_feat / img_feat.norm(dim=-1, keepdim=True)
sim_pos = img_feat @ self.pos_feats.T # (1, P)
sim_neg = img_feat @ self.neg_feats.T # (1, N)
logits = torch.cat([sim_pos * 100, sim_neg * 100], dim=1)
prob = logits.softmax(dim=1)[0, : len(CLIP_POSITIVE)].max().item()
return prob
def dino_detect(self, img: Image.Image):
"""Grounding DINO 检测,返回 [x1,y1,x2,y2] 绝对坐标列表"""
inputs = self.proc(images=img, text=DINO_PROMPTS, return_tensors="pt")
with torch.no_grad():
out = self.dino(**inputs)
res = self.proc.post_process_grounded_object_detection(
out, input_ids=inputs["input_ids"],
threshold=DINO_BOX_THRESHOLD, text_threshold=DINO_TEXT_THRESHOLD,
target_sizes=[img.size[::-1]],
)[0]
w, h = img.size
boxes = res["boxes"].tolist()
scores = res["scores"].tolist()
valid = []
for (x1, y1, x2, y2), s in zip(boxes, scores):
bw, bh = (x2 - x1) / w, (y2 - y1) / h
if bw * bh < MIN_BOX_AREA:
continue
valid.append((x1, y1, x2, y2, s))
return valid
def clip_fallback(self, img: Image.Image):
"""滑动窗口找最像野鸡的区域(兜底),返回 [x1,y1,x2,y2] 或 None"""
w, h = img.size
base = 768
scale = base / max(w, h)
small = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
sw, sh = small.size
best = None
best_score = 0.0
for f in FALLBACK_WINDOW_SIZES:
ws, hs = int(sw * f), int(sh * f)
stride = max(int(ws * 0.5), 8)
for y in range(0, max(sh - hs, 1), stride):
for x in range(0, max(sw - ws, 1), stride):
region = small.crop((x, y, x + ws, y + hs))
s = self.clip_score(region)
if s > best_score:
best_score = s
best = (x, y, x + ws, y + hs)
if best is None:
return None, best_score
x1, y1, x2, y2 = [v / scale for v in best]
return (x1, y1, x2, y2), best_score
def label_image(self, img: Image.Image, name: str):
"""返回 (boxes, source, debug)boxes 为 [x1,y1,x2,y2] 绝对坐标列表"""
w, h = img.size
cands = self.dino_detect(img)
if cands:
boxes = torch.tensor([[c[0], c[1], c[2], c[3]] for c in cands], dtype=torch.float32)
scores = torch.tensor([c[4] for c in cands], dtype=torch.float32)
keep = nms(boxes, scores, NMS_IOU)
kept = [(cands[i][0], cands[i][1], cands[i][2], cands[i][3], cands[i][4]) for i in keep]
# CLIP 验证每个框
final = []
for x1, y1, x2, y2, s in kept:
pad_x, pad_y = (x2 - x1) * 0.2, (y2 - y1) * 0.2
region = img.crop((max(x1 - pad_x, 0), max(y1 - pad_y, 0),
min(x2 + pad_x, w), min(y2 + pad_y, h)))
prob = self.clip_score(region)
final.append((x1, y1, x2, y2, s, prob))
best_clip = max(p for _, _, _, _, _, p in final)
if best_clip < CLIP_VERIFY_THRESHOLD:
print(f" {name}: 整图框分均过低,改用 CLIP 兜底 (best_clip={best_clip:.2f})", flush=True)
box, score = self.clip_fallback(img)
if box is not None:
x1, y1, x2, y2 = box
return [(x1, y1, x2, y2, score, f"clip({score:.2f})")], "clip"
return [], "none"
kept_final = [f for f in final if f[5] >= 0.20]
if not kept_final:
kept_final = [max(final, key=lambda t: t[5])]
for x1, y1, x2, y2, s, p in final:
if (x1, y1, x2, y2, s, p) not in kept_final:
print(f" {name}: 剔除低分框 clip={p:.2f} [{x1:.0f},{y1:.0f},{x2:.0f},{y2:.0f}]", flush=True)
return [(x1, y1, x2, y2, s, f"dino(clip={p:.2f})")
for x1, y1, x2, y2, s, p in kept_final], "dino"
else:
print(f" {name}: DINO 无候选,使用 CLIP 兜底", flush=True)
# 兜底:CLIP 滑动窗口
box, score = self.clip_fallback(img)
if box is not None:
x1, y1, x2, y2 = box
return [(x1, y1, x2, y2, score, f"clip({score:.2f})")], "clip"
return [], "none"
def main():
(OUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
(OUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
PREVIEW_DIR.mkdir(parents=True, exist_ok=True)
labeler = AutoLabeler()
files = sorted(f for f in IMG_DIR.iterdir() if f.suffix.lower() in {".png", ".jpg", ".jpeg"})
print(f"{len(files)} 张图片", flush=True)
summary = []
for p in files:
img = Image.open(p).convert("RGB")
w, h = img.size
boxes, src = labeler.label_image(img, p.name)
summary.append((p.name, len(boxes), src))
label_path = OUT_DIR / "labels" / "train" / (p.stem + ".txt")
img.save(OUT_DIR / "images" / "train" / p.name)
if boxes:
with open(label_path, "w") as f:
for x1, y1, x2, y2, s, _ in boxes:
cx, cy = (x1 + x2) / 2 / w, (y1 + y2) / 2 / h
bw, bh = (x2 - x1) / w, (y2 - y1) / h
f.write(f"0 {cx:.6f} {cy:.6f} {bw:.6f} {bh:.6f}\n")
# 预览图
draw = ImageDraw.Draw(img)
for x1, y1, x2, y2, s, tag in boxes:
draw.rectangle([x1, y1, x2, y2], outline="#E53935", width=4)
draw.text((x1 + 4, max(y1 - 18, 0)), f"pheasant {s:.2f} [{tag}]", fill="#E53935")
img.save(PREVIEW_DIR / (p.stem + ".jpg"), "JPEG", quality=85)
status = "" if boxes else ""
print(f"{status} {p.name}: {len(boxes)} 框 ({src})", flush=True)
# 生成 data.yaml
with open(OUT_DIR / "data.yaml", "w") as f:
f.write(f"path: {OUT_DIR}\n")
f.write("train: images/train\nval: images/train\n")
f.write("nc: 1\nnames: ['pheasant']\n")
labeled = sum(1 for _, n, _ in summary if n > 0)
print(f"\n完成: {labeled}/{len(summary)} 张已标注", flush=True)
for name, n, src in summary:
print(f" {name}: {n} 框 ({src})", flush=True)
if __name__ == "__main__":
sys.exit(main())
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""
使用 CLIP 模型的滑动窗口方法检测野鸡位置
生成 YOLO 格式的标注文件
"""
import cv2
import numpy as np
import torch
import clip
from PIL import Image
from pathlib import Path
from torchvision.ops import nms
# 配置
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
# 检测配置
WINDOW_SIZES = [128, 256, 512] # 滑动窗口大小(增大)
STRIDE_RATIO = 0.7 # 窗口滑动步长比例(增大)
CONFIDENCE_THRESHOLD = 0.5 # 置信度阈值(提高)
NMS_THRESHOLD = 0.5 # NMS 阈值(提高)
# 野鸡的文本描述
PHEASANT_PROMPTS = [
"a photo of a pheasant in the wild",
"a wild pheasant in natural habitat",
"a bird with colorful feathers in grass",
]
class CLIPDetector:
def __init__(self):
print("加载 CLIP 模型...")
self.model, self.preprocess = clip.load("ViT-B/32", device="cpu")
self.model.eval()
print("CLIP 模型加载完成")
# 预计算文本特征
print("预计算文本特征...")
text_tokens = clip.tokenize(PHEASANT_PROMPTS).to("cpu")
with torch.no_grad():
self.text_features = self.model.encode_text(text_tokens)
self.text_features /= self.text_features.norm(dim=-1, keepdim=True)
def classify_region(self, image_region):
"""对图像区域进行分类"""
# 转换为 PIL Image
if isinstance(image_region, np.ndarray):
image_region = Image.fromarray(cv2.cvtColor(image_region, cv2.COLOR_BGR2RGB))
# 预处理
image_input = self.preprocess(image_region).unsqueeze(0).to("cpu")
# 计算图像特征
with torch.no_grad():
image_features = self.model.encode_image(image_input)
image_features /= image_features.norm(dim=-1, keepdim=True)
# 计算相似度
similarity = (100.0 * image_features @ self.text_features.T).softmax(dim=-1)
# 返回最高分
return float(similarity[0].max())
def detect(self, image):
"""检测图片中的野鸡"""
h, w = image.shape[:2]
detections = []
# 多尺度滑动窗口
for window_size in WINDOW_SIZES:
stride = int(window_size * STRIDE_RATIO)
# 滑动窗口
for y in range(0, h - window_size, stride):
for x in range(0, w - window_size, stride):
# 提取窗口区域
region = image[y:y+window_size, x:x+window_size]
# 分类
score = self.classify_region(region)
# 如果置信度足够高,保存检测结果
if score > CONFIDENCE_THRESHOLD:
detections.append({
'bbox': [x, y, x + window_size, y + window_size],
'score': score,
})
# NMS 去重
if len(detections) > 0:
detections = self.nms(detections)
return detections
def nms(self, detections):
"""非极大值抑制"""
if len(detections) == 0:
return []
# 转换为 torch 格式
boxes = torch.tensor([d['bbox'] for d in detections], dtype=torch.float32)
scores = torch.tensor([d['score'] for d in detections], dtype=torch.float32)
# 应用 NMS
keep_indices = nms(boxes, scores, NMS_THRESHOLD)
# 保留 NMS 后的检测结果
filtered_detections = [detections[i] for i in keep_indices]
return filtered_detections
def convert_to_yolo_format(bbox, img_width, img_height):
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
x1, y1, x2, y2 = bbox
cx = (x1 + x2) / 2 / img_width
cy = (y1 + y2) / 2 / img_height
w = (x2 - x1) / img_width
h = (y2 - y1) / img_height
return cx, cy, w, h
def main():
# 初始化检测器
detector = CLIPDetector()
# 创建输出目录
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
# 获取所有图片
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
image_files = [
f for f in IMAGES_DIR.iterdir()
if f.suffix.lower() in image_extensions
]
print(f"找到 {len(image_files)} 张图片")
# 处理每张图片
labeled_count = 0
for img_path in image_files:
print(f"\n处理: {img_path.name}")
# 读取图片
image = cv2.imread(str(img_path))
if image is None:
print(f" 跳过: 无法读取 {img_path.name}")
continue
img_height, img_width = image.shape[:2]
# 检测野鸡
detections = detector.detect(image)
# 生成标注
labels = []
for det in detections:
bbox = det['bbox']
score = det['score']
# 转换为 YOLO 格式
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
labels.append(f"0 {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
# 在图片上绘制检测框(用于可视化)
x1, y1, x2, y2 = bbox
cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
cv2.putText(image, f"pheasant: {score:.2f}", (int(x1), int(y1) - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
# 保存标注文件
if len(labels) > 0:
labeled_count += 1
# 保存带标注的图片(用于可视化)
cv2.imwrite(str(OUTPUT_DIR / "images" / "train" / img_path.name), image)
# 保存标注
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
with open(label_path, "w") as f:
f.writelines(labels)
print(f" ✓ 标注了 {len(labels)} 个目标")
else:
print(f" - 未检测到野鸡")
print(f"\n完成!")
print(f" 总图片数: {len(image_files)}")
print(f" 有效标注: {labeled_count}")
print(f" 输出目录: {OUTPUT_DIR}")
# 创建 data.yaml
data_yaml = OUTPUT_DIR / "data.yaml"
with open(data_yaml, "w") as f:
f.write(f"""# Observer 数据集配置 - 只识别野鸡
path: {OUTPUT_DIR}
train: images/train
val: images/train
# 类别
nc: 1
names: ['pheasant']
""")
print(f" 数据配置: {data_yaml}")
if __name__ == "__main__":
main()
+300
View File
@@ -0,0 +1,300 @@
"""从 Wikimedia Commons 下载训练图片(自由版权,可离线使用)。
策略:优先用物种分类目录(图片内容精确),再用全文搜索补充场景/姿态/光线多样性。
输出: datasets/images/<class>/<index>.jpg
串行 + 失败重试 + pHash 视觉去重。
"""
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import imagehash
import numpy as np
from PIL import Image
API = "https://commons.wikimedia.org/w/api.php"
OUT = os.path.join(os.path.dirname(__file__), "datasets", "images")
# 更多图片来源
ADDITIONAL_SOURCES = {
"inaturalist": "https://api.inaturalist.org/v1/observations",
"flickr": "https://api.flickr.com/services/rest/",
}
SOURCES = {
"pheasant": {
"categories": ["Phasianus colchicus"],
"queries": [
# 户外真实场景(重点!)
"pheasant in field",
"pheasant in grassland",
"pheasant in meadow",
"pheasant in farmland",
"pheasant in countryside",
"pheasant in wild",
# 部分遮挡场景(重点!实际使用场景)
"pheasant hiding in grass",
"pheasant hiding in bushes",
"pheasant partially hidden",
"pheasant behind vegetation",
"pheasant peeking through grass",
"pheasant concealed in foliage",
"pheasant camouflaged",
"pheasant blending in",
# 不同距离和角度
"pheasant distant view",
"pheasant far away",
"pheasant small in frame",
"pheasant side view",
"pheasant back view",
"pheasant from behind",
# 不同姿态和行为
"pheasant walking in grass",
"pheasant foraging",
"pheasant feeding",
"pheasant running",
"pheasant flying low",
# 光线条件(户外真实光线)
"pheasant natural light",
"pheasant daylight",
"pheasant shade",
"pheasant shadow",
"pheasant backlit",
"pheasant overcast",
# 季节环境
"pheasant in autumn",
"pheasant in winter",
"pheasant in spring",
"pheasant in summer",
"pheasant in dry grass",
"pheasant in green grass",
"pheasant in snow",
"pheasant in mud",
],
},
}
PER_CLASS = 80
MIN_PIXEL = 320
RETRY = 2
WORKERS = 1
SLEEP_S = 2.0
RATE_LIMIT_WAIT_S = 60
def api_request(params: dict) -> dict:
url = API + "?" + urllib.parse.urlencode(params)
for attempt in range(RETRY + 1):
try:
req = urllib.request.Request(url, headers={"User-Agent": "observer-training/1.0"})
with urllib.request.urlopen(req, timeout=30) as r:
return json.loads(r.read().decode("utf-8"))
except Exception:
if attempt == RETRY:
raise
time.sleep(2 * (attempt + 1))
def collect_pages(pages: dict, results: list) -> None:
for p in pages.values():
info = (p.get("imageinfo") or [{}])[0]
thumb = info.get("thumburl")
if not thumb:
continue
w, h = info.get("width", 0), info.get("height", 0)
if min(w, h) < MIN_PIXEL:
continue
results.append({"url": thumb, "w": w, "h": h, "title": p.get("title", "")})
def category_images(category: str, limit: int) -> list[dict]:
results = []
params = {
"action": "query",
"generator": "categorymembers",
"gcmtitle": f"Category:{category}",
"gcmtype": "file",
"gcmlimit": "50",
"prop": "imageinfo",
"iiprop": "url|size",
"iiurlwidth": "640",
"format": "json",
}
while len(results) < limit:
data = api_request(params)
pages = (data.get("query", {}) or {}).get("pages", {})
collect_pages(pages, results)
cont = (data.get("continue") or {}).get("gcmcontinue")
if not cont:
break
params["gcmcontinue"] = cont
return results[:limit]
def search_images(query: str, limit: int) -> list[dict]:
results = []
params = {
"action": "query",
"generator": "search",
"gsrsearch": f"filetype:bitmap {query}",
"gsrnamespace": "6",
"gsrlimit": str(limit),
"prop": "imageinfo",
"iiprop": "url|size",
"iiurlwidth": "640",
"format": "json",
}
data = api_request(params)
pages = (data.get("query", {}) or {}).get("pages", {})
collect_pages(pages, results)
return results
def is_text_heavy(image_path: str) -> bool:
"""检测图片是否包含大量文字或为图表/标志"""
try:
img = Image.open(image_path).convert("L") # 灰度
arr = np.array(img, dtype=np.float32)
# 1. 检测边缘密度(文字产生大量边缘)
# 简单边缘检测:计算像素梯度
dx = np.abs(np.diff(arr, axis=1))
dy = np.abs(np.diff(arr, axis=0))
edge_density = (np.mean(dx) + np.mean(dy)) / 2
# 2. 检测颜色方差(图表通常颜色单一)
# 转回RGB检查
img_rgb = Image.open(image_path).convert("RGB")
arr_rgb = np.array(img_rgb, dtype=np.float32)
color_variance = np.std(arr_rgb)
# 3. 检测对比度(文字通常有高对比度的边缘)
contrast = np.std(arr)
# 判断逻辑:
# - 高边缘密度 + 低颜色方差 = 可能是图表/标志
# - 高对比度 + 高边缘密度 = 可能是文字图片
if edge_density > 30 and color_variance < 50:
return True
if edge_density > 40 and contrast > 80:
return True
return False
except Exception:
return False
def download(url: str, path: str) -> bool:
for attempt in range(RETRY + 1):
try:
req = urllib.request.Request(url, headers={"User-Agent": "observer-training/1.0"})
with urllib.request.urlopen(req, timeout=60) as r:
data = r.read()
if "image/" not in r.headers.get("Content-Type", ""):
return False
if len(data) < 10_000:
return False
with open(path, "wb") as f:
f.write(data)
return True
except urllib.error.HTTPError as e:
if e.code == 429:
# 限流:冷却后重试
print(f" ⏳ 限流(429),冷却 {RATE_LIMIT_WAIT_S}s", flush=True)
time.sleep(RATE_LIMIT_WAIT_S)
continue
if attempt == RETRY:
return False
time.sleep(2 * (attempt + 1))
except Exception:
if attempt == RETRY:
return False
time.sleep(2 * (attempt + 1))
def main():
for cls, src in SOURCES.items():
cls_dir = os.path.join(OUT, cls)
os.makedirs(cls_dir, exist_ok=True)
existing = len([f for f in os.listdir(cls_dir) if f.endswith(".jpg")])
if existing >= PER_CLASS:
print(f"[skip] {cls}: 已有 {existing}", flush=True)
continue
# 重建去重池(续跑兼容)
seen_hashes = []
for f in os.listdir(cls_dir):
if f.endswith(".jpg"):
try:
seen_hashes.append(imagehash.phash(Image.open(os.path.join(cls_dir, f))))
except Exception:
pass
candidates = []
seen_titles = set()
for cat in src["categories"]:
print(f"[category] {cls} <- {cat}", flush=True)
try:
for item in category_images(cat, 80):
if item["title"] in seen_titles:
continue
seen_titles.add(item["title"])
candidates.append(item)
except Exception as e:
print(f" ! category 失败: {e}", flush=True)
for query in src["queries"]:
if len(candidates) >= PER_CLASS * 2:
break
print(f"[fetch] {cls} <- \"{query}\"", flush=True)
try:
for item in search_images(query, 80):
if item["title"] in seen_titles:
continue
seen_titles.add(item["title"])
candidates.append(item)
except Exception as e:
print(f" ! 搜索失败: {e}", flush=True)
print(f"[download] {cls}: 候选 {len(candidates)}", flush=True)
saved = existing
fail = 0
dup = 0
for item in candidates:
if saved >= PER_CLASS:
break
saved += 1
path = os.path.join(cls_dir, f"{saved:03d}.jpg")
if not download(item["url"], path):
fail += 1
saved -= 1
continue
# 过滤文字/图表类图片
if is_text_heavy(path):
os.remove(path)
saved -= 1
continue
try:
h = imagehash.phash(Image.open(path))
if any(h - other <= 8 for other in seen_hashes):
os.remove(path)
dup += 1
saved -= 1
continue
seen_hashes.append(h)
except Exception:
pass
if saved % 10 == 0:
print(f" + {cls}: {saved}", flush=True)
time.sleep(SLEEP_S)
print(f"[done] {cls}: {saved} 张(失败 {fail},重复 {dup}", flush=True)
if __name__ == "__main__":
sys.exit(main())
+112
View File
@@ -0,0 +1,112 @@
"""用 LocalAI qwen3.5-9b 过滤不含活体动物的图片。
用法:python filter_images.py
输入:datasets/images/<class>/*.jpg
输出:删除不含目标动物的图片
"""
import os
import json
import base64
import sys
import urllib.request
from io import BytesIO
from PIL import Image
# LocalAI 服务器配置
LOCAL_AI_URL = "http://192.168.3.210:18080/v1/chat/completions"
MODEL_NAME = "qwen3.5-9b"
# 过滤配置
BASE = os.path.dirname(__file__)
IMG_DIR = os.path.join(BASE, "datasets", "images")
CLASSES = ["pheasant"]
MAX_IMAGE_SIZE = 400 # 缩放到最大边长
# 过滤提示词
FILTER_PROMPT = """你是一个图片质量检查助手。请判断这张图片是否包含【{target}】的活体动物照片。
判断标准:
✓ 保留:真实动物照片(活体、自然姿态、野外或自然环境)
✗ 删除:标本照片、插画、图表、文字图片、logo、标志、空场景、纯风景
只回答:KEEP 或 DELETE"""
def encode_image_to_base64(path: str) -> str:
"""读取图片并压缩后转为 base64"""
img = Image.open(path).convert("RGB") # 转为 RGB 避免 RGBA 问题
# 缩放图片以减少 API 负载
img.thumbnail((MAX_IMAGE_SIZE, MAX_IMAGE_SIZE), Image.Resampling.LANCZOS)
buffer = BytesIO()
img.save(buffer, format="JPEG", quality=70)
return base64.b64encode(buffer.getvalue()).decode("utf-8")
def call_localai(image_path: str, prompt: str) -> str:
"""调用 LocalAI 检测图片是否包含活体动物"""
image_b64 = encode_image_to_base64(image_path)
data = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
],
}
],
"temperature": 0.1,
}
for attempt in range(3):
try:
with urllib.request.urlopen(LOCAL_AI_URL, data=json.dumps(data).encode("utf-8"), timeout=60) as resp:
if resp.status == 200:
result = json.loads(resp.read().decode("utf-8"))
return result.get("choices", [{}])[0].get("message", {}).get("content", "").strip().upper()
except Exception as e:
if attempt < 2:
import time
time.sleep(2 * (attempt + 1))
else:
print(f" ! API 错误: {e}", flush=True)
return "ERROR"
def main():
total_deleted = 0
for cls in CLASSES:
cls_dir = os.path.join(IMG_DIR, cls)
if not os.path.exists(cls_dir):
continue
files = sorted(f for f in os.listdir(cls_dir) if f.endswith(".jpg"))
print(f"\n[filter] {cls}: {len(files)}", flush=True)
deleted = 0
kept = 0
for f in files:
path = os.path.join(cls_dir, f)
prompt = FILTER_PROMPT.format(target=cls)
result = call_localai(path, prompt)
if "DELETE" in result:
os.remove(path)
deleted += 1
print(f" {f}: DELETE", flush=True)
else:
kept += 1
print(f" {f}: KEEP", flush=True)
print(f" → 保留 {kept} 张,删除 {deleted}", flush=True)
total_deleted += deleted
print(f"\n[done] 共删除 {total_deleted} 张图片", flush=True)
if __name__ == "__main__":
sys.exit(main())
+130
View File
@@ -0,0 +1,130 @@
"""用本地 CLIP 模型过滤不含活体动物的图片。
用法:python filter_images_clip.py
输入:datasets/images/<class>/*.jpg
输出:删除不含目标动物的图片
"""
import os
import sys
import torch
import clip
from PIL import Image
# 过滤配置
BASE = os.path.dirname(__file__)
IMG_DIR = os.path.join(BASE, "datasets", "images")
CLASSES = ["pheasant"]
MODEL_PATH = os.path.join(BASE, "..", "weights", "clip", "ViT-B-32.pt")
# 判断阈值 - 相似度低于此值的图片将被删除
SIMILARITY_THRESHOLD = 0.10
# 每个类别的正向和负向描述
PROMPTS = {
"pheasant": {
"positive": [
"a photo of a pheasant in the wild",
"a wild pheasant in natural outdoor habitat",
"a pheasant walking in grass or field",
"a pheasant hiding in bushes",
"a pheasant in natural environment",
],
"negative": [
"a taxidermy pheasant",
"an illustration of a pheasant",
"a drawing of a bird",
"a painting of a bird",
"a cartoon of a bird",
"a person holding a bird",
"a person catching a bird",
"a bird in a cage",
"a bird indoors",
"a bird in a house",
"a bird in a zoo",
"a bird in captivity",
"a logo or icon",
"text or writing",
"a landscape without animals",
"a statue or sculpture",
"a stuffed animal",
],
},
}
def main():
# 加载 CLIP 模型
print("[init] 加载 CLIP 模型...", flush=True)
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device, download_root=os.path.join(BASE, "..", "weights"))
# 缓存文本特征
text_features_cache = {}
for cls in CLASSES:
if cls not in PROMPTS:
continue
pos_texts = clip.tokenize(PROMPTS[cls]["positive"]).to(device)
neg_texts = clip.tokenize(PROMPTS[cls]["negative"]).to(device)
with torch.no_grad():
pos_features = model.encode_text(pos_texts)
neg_features = model.encode_text(neg_texts)
# 取正向描述的平均特征
pos_features = pos_features.mean(dim=0, keepdim=True)
pos_features /= pos_features.norm(dim=-1, keepdim=True)
neg_features = neg_features.mean(dim=0, keepdim=True)
neg_features /= neg_features.norm(dim=-1, keepdim=True)
text_features_cache[cls] = (pos_features, neg_features)
total_deleted = 0
for cls in CLASSES:
cls_dir = os.path.join(IMG_DIR, cls)
if not os.path.exists(cls_dir):
continue
files = sorted(f for f in os.listdir(cls_dir) if f.endswith(".jpg"))
print(f"\n[filter] {cls}: {len(files)}", flush=True)
deleted = 0
kept = 0
pos_features, neg_features = text_features_cache[cls]
for f in files:
path = os.path.join(cls_dir, f)
try:
image = preprocess(Image.open(path).convert("RGB")).unsqueeze(0).to(device)
with torch.no_grad():
image_features = model.encode_image(image)
image_features /= image_features.norm(dim=-1, keepdim=True)
# 计算与正向和负向描述的相似度
pos_similarity = (image_features @ pos_features.T).item()
neg_similarity = (image_features @ neg_features.T).item()
# 综合得分:正向相似度 - 负向相似度
score = pos_similarity - neg_similarity
if score < SIMILARITY_THRESHOLD:
os.remove(path)
deleted += 1
print(f" {f}: DELETE (score={score:.3f})", flush=True)
else:
kept += 1
print(f" {f}: KEEP (score={score:.3f})", flush=True)
except Exception as e:
print(f" {f}: ERROR ({e})", flush=True)
kept += 1 # 出错时保留
print(f" → 保留 {kept} 张,删除 {deleted}", flush=True)
total_deleted += deleted
print(f"\n[done] 共删除 {total_deleted} 张图片", flush=True)
if __name__ == "__main__":
sys.exit(main())
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""
训练 YOLOv8n 模型 - 只识别野鸡(pheasant)
使用现有的训练数据进行迁移学习
"""
from pathlib import Path
from ultralytics import YOLO
# 配置
DATA_DIR = Path(__file__).parent / "datasets"
MODEL_NAME = "yolov8n.pt" # 预训练模型
OUTPUT_DIR = Path(__file__).parent / "runs"
def main():
# 创建数据集配置文件(只包含野鸡类别)
data_yaml = DATA_DIR / "data.yaml"
print("创建数据集配置文件(只训练野鸡类别)...")
with open(data_yaml, "w") as f:
f.write(f"""# Observer 数据集配置 - 只识别野鸡
path: {DATA_DIR}
train: images/pheasant
val: images/pheasant
# 类别
nc: 1
names: ['pheasant']
""")
# 加载预训练模型
print(f"加载预训练模型: {MODEL_NAME}")
model = YOLO(MODEL_NAME)
# 训练模型
print("开始训练...")
results = model.train(
data=str(data_yaml),
epochs=50,
imgsz=640,
batch=16,
name="observer_yolov8n",
patience=20,
save=True,
plots=True
)
print(f"\n训练完成!")
print(f"最佳模型保存在: {OUTPUT_DIR / 'observer_yolov8n' / 'weights' / 'best.pt'}")
# 导出为 TFLite 格式
print("\n导出为 TFLite 格式...")
best_model_path = OUTPUT_DIR / "observer_yolov8n" / "weights" / "best.pt"
if best_model_path.exists():
best_model = YOLO(str(best_model_path))
best_model.export(format="tflite", imgsz=320)
print(f"TFLite 模型导出完成")
if __name__ == "__main__":
main()
+154
View File
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""
使用 Hugging Face transformers 进行零样本目标检测
使用 OWL-ViT 模型检测野鸡
"""
import cv2
import torch
from pathlib import Path
from transformers import OwlViTProcessor, OwlViTForObjectDetection
from PIL import Image
# 配置
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
# 检测配置
CONFIDENCE_THRESHOLD = 0.1 # 置信度阈值
# 文本描述
TEXT_PROMPTS = ["pheasant", "wild bird", "bird in grass"]
class ZeroShotDetector:
def __init__(self):
print("加载 OWL-ViT 模型...")
self.processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32")
self.model = OwlViTForObjectDetection.from_pretrained("google/owlvit-base-patch32")
self.model.eval()
print("OWL-ViT 模型加载完成")
def detect(self, image_path):
"""检测图片中的野鸡"""
# 读取图片
image = Image.open(image_path).convert("RGB")
# 准备输入
inputs = self.processor(text=TEXT_PROMPTS, images=image, return_tensors="pt")
# 推理
with torch.no_grad():
outputs = self.model(**inputs)
# 获取结果
target_sizes = torch.tensor([image.size[::-1]]) # [height, width]
results = self.processor.post_process_grounded_object_detection(
outputs, threshold=CONFIDENCE_THRESHOLD, target_sizes=target_sizes
)[0]
# 解析结果
detections = []
for score, label, box in zip(results["scores"], results["labels"], results["boxes"]):
box = box.tolist()
detections.append({
'bbox': box, # [x1, y1, x2, y2]
'score': score.item(),
'label': TEXT_PROMPTS[label],
})
return detections
def convert_to_yolo_format(bbox, img_width, img_height):
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
x1, y1, x2, y2 = bbox
cx = (x1 + x2) / 2 / img_width
cy = (y1 + y2) / 2 / img_height
w = (x2 - x1) / img_width
h = (y2 - y1) / img_height
return cx, cy, w, h
def main():
# 初始化检测器
detector = ZeroShotDetector()
# 创建输出目录
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
# 获取所有图片
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
image_files = [
f for f in IMAGES_DIR.iterdir()
if f.suffix.lower() in image_extensions
]
print(f"找到 {len(image_files)} 张图片")
# 处理每张图片
labeled_count = 0
for img_path in image_files:
print(f"\n处理: {img_path.name}")
# 读取图片
image = cv2.imread(str(img_path))
if image is None:
print(f" 跳过: 无法读取 {img_path.name}")
continue
img_height, img_width = image.shape[:2]
# 检测野鸡
detections = detector.detect(img_path)
# 生成标注
labels = []
for det in detections:
bbox = det['bbox']
score = det['score']
# 转换为 YOLO 格式
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
labels.append(f"0 {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
# 在图片上绘制检测框(用于可视化)
x1, y1, x2, y2 = bbox
cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
cv2.putText(image, f"pheasant: {score:.2f}", (int(x1), int(y1) - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
# 保存标注文件
if len(labels) > 0:
labeled_count += 1
# 保存带标注的图片(用于可视化)
cv2.imwrite(str(OUTPUT_DIR / "images" / "train" / img_path.name), image)
# 保存标注
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
with open(label_path, "w") as f:
f.writelines(labels)
print(f" ✓ 标注了 {len(labels)} 个目标")
else:
print(f" - 未检测到野鸡")
print(f"\n完成!")
print(f" 总图片数: {len(image_files)}")
print(f" 有效标注: {labeled_count}")
print(f" 输出目录: {OUTPUT_DIR}")
# 创建 data.yaml
data_yaml = OUTPUT_DIR / "data.yaml"
with open(data_yaml, "w") as f:
f.write(f"""# Observer 数据集配置 - 只识别野鸡
path: {OUTPUT_DIR}
train: images/train
val: images/train
# 类别
nc: 1
names: ['pheasant']
""")
print(f" 数据配置: {data_yaml}")
if __name__ == "__main__":
main()