初始化 observer 项目:纯代码,不含权重与训练数据
This commit is contained in:
+12
@@ -0,0 +1,12 @@
|
||||
*.iml
|
||||
.gradle/
|
||||
local.properties
|
||||
.idea/
|
||||
build/
|
||||
captures/
|
||||
.externalNativeBuild/
|
||||
.cxx/
|
||||
.DS_Store
|
||||
training/datasets/
|
||||
training/venv/
|
||||
*.pt
|
||||
@@ -0,0 +1,80 @@
|
||||
plugins {
|
||||
alias(libs.plugins.android.application)
|
||||
alias(libs.plugins.kotlin.android)
|
||||
alias(libs.plugins.kotlin.compose)
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "com.example.observer"
|
||||
compileSdk = 35
|
||||
|
||||
defaultConfig {
|
||||
applicationId = "com.example.observer"
|
||||
minSdk = 24
|
||||
targetSdk = 35
|
||||
versionCode = 1
|
||||
versionName = "1.0"
|
||||
|
||||
ndk {
|
||||
abiFilters += listOf("arm64-v8a", "armeabi-v7a")
|
||||
}
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = true
|
||||
isShrinkResources = true
|
||||
proguardFiles(
|
||||
getDefaultProguardFile("proguard-android-optimize.txt"),
|
||||
"proguard-rules.pro",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
kotlinOptions {
|
||||
jvmTarget = "17"
|
||||
}
|
||||
|
||||
buildFeatures {
|
||||
compose = true
|
||||
}
|
||||
|
||||
packaging {
|
||||
resources {
|
||||
excludes += "/META-INF/{AL2.0,LGPL2.1}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation(libs.androidx.core.ktx)
|
||||
implementation(libs.androidx.lifecycle.runtime.ktx)
|
||||
implementation(libs.androidx.lifecycle.viewmodel.compose)
|
||||
implementation(libs.androidx.lifecycle.runtime.compose)
|
||||
implementation(libs.androidx.activity.compose)
|
||||
|
||||
implementation(platform(libs.androidx.compose.bom))
|
||||
implementation(libs.androidx.compose.ui)
|
||||
implementation(libs.androidx.compose.ui.graphics)
|
||||
implementation(libs.androidx.compose.ui.tooling.preview)
|
||||
implementation(libs.androidx.compose.material3)
|
||||
debugImplementation(libs.androidx.compose.ui.tooling)
|
||||
|
||||
implementation(libs.androidx.camera.core)
|
||||
implementation(libs.androidx.camera.camera2)
|
||||
implementation(libs.androidx.camera.lifecycle)
|
||||
implementation(libs.androidx.camera.view)
|
||||
|
||||
implementation(libs.org.tensorflow.lite)
|
||||
implementation(libs.org.tensorflow.lite.gpu)
|
||||
|
||||
implementation(libs.androidx.datastore.preferences)
|
||||
implementation(libs.kotlinx.coroutines.android)
|
||||
|
||||
testImplementation(libs.junit)
|
||||
}
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
# TensorFlow Lite
|
||||
-keep class org.tensorflow.** { *; }
|
||||
-dontwarn org.tensorflow.**
|
||||
@@ -0,0 +1,26 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-feature android:name="android.hardware.camera.any" android:required="true" />
|
||||
|
||||
<application
|
||||
android:name=".ObserverApp"
|
||||
android:allowBackup="false"
|
||||
android:icon="@drawable/ic_launcher"
|
||||
android:label="@string/app_name"
|
||||
android:supportsRtl="true"
|
||||
android:theme="@style/Theme.Observer">
|
||||
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true"
|
||||
android:screenOrientation="portrait"
|
||||
android:configChanges="orientation|screenSize|keyboardHidden">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -0,0 +1,4 @@
|
||||
pheasant
|
||||
hare
|
||||
dove
|
||||
fish
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.example.observer
|
||||
|
||||
import android.os.Bundle
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import com.example.observer.ui.ObserverApp
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
enableEdgeToEdge()
|
||||
super.onCreate(savedInstanceState)
|
||||
setContent {
|
||||
ObserverApp()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.example.observer
|
||||
|
||||
import android.app.Application
|
||||
import android.content.Context
|
||||
import com.example.observer.data.SettingsRepository
|
||||
import com.example.observer.detection.Detector
|
||||
import com.example.observer.detection.TFLiteDetector
|
||||
import com.example.observer.distance.DistanceEstimator
|
||||
import com.example.observer.reminder.Reminder
|
||||
|
||||
class ObserverApp : Application() {
|
||||
lateinit var container: AppContainer
|
||||
private set
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
container = AppContainer(this)
|
||||
}
|
||||
}
|
||||
|
||||
class AppContainer(context: Context) {
|
||||
val settingsRepository = SettingsRepository(context)
|
||||
val detector: Detector? = TFLiteDetector.create(context)
|
||||
val distanceEstimator = DistanceEstimator(context)
|
||||
val reminder = Reminder(context)
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package com.example.observer.camera
|
||||
|
||||
import android.util.Log
|
||||
import androidx.camera.core.Camera
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.camera2.interop.Camera2CameraInfo
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import java.util.concurrent.Executor
|
||||
|
||||
class CameraController(
|
||||
private val lifecycleOwner: LifecycleOwner,
|
||||
private val analysisExecutor: Executor,
|
||||
) {
|
||||
var onCameraIdChanged: ((String) -> Unit)? = null
|
||||
|
||||
var onInitError: (() -> Unit)? = null
|
||||
|
||||
var currentCameraId: String? = null
|
||||
private set
|
||||
|
||||
private var cameraProvider: ProcessCameraProvider? = null
|
||||
private var isFrontFacing = false
|
||||
|
||||
fun start(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
|
||||
val future = ProcessCameraProvider.getInstance(previewView.context)
|
||||
future.addListener({
|
||||
try {
|
||||
val provider = future.get()
|
||||
cameraProvider = provider
|
||||
bind(previewView, analyzer)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "相机初始化失败", e)
|
||||
onInitError?.invoke()
|
||||
}
|
||||
}, ContextCompat.getMainExecutor(previewView.context))
|
||||
}
|
||||
|
||||
fun switchCamera(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
|
||||
isFrontFacing = !isFrontFacing
|
||||
bind(previewView, analyzer)
|
||||
}
|
||||
|
||||
private fun bind(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer?) {
|
||||
val provider = cameraProvider ?: return
|
||||
provider.unbindAll()
|
||||
|
||||
val preview = Preview.Builder().build().also {
|
||||
it.surfaceProvider = previewView.surfaceProvider
|
||||
}
|
||||
val imageAnalysis = ImageAnalysis.Builder()
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
|
||||
.build()
|
||||
analyzer?.let { imageAnalysis.setAnalyzer(analysisExecutor, it) }
|
||||
|
||||
val camera: Camera = provider.bindToLifecycle(
|
||||
lifecycleOwner,
|
||||
currentSelector(),
|
||||
preview,
|
||||
imageAnalysis,
|
||||
)
|
||||
currentCameraId = try {
|
||||
Camera2CameraInfo.from(camera.cameraInfo).cameraId
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "获取 cameraId 失败", e)
|
||||
null
|
||||
}
|
||||
currentCameraId?.let { onCameraIdChanged?.invoke(it) }
|
||||
}
|
||||
|
||||
private fun currentSelector(): CameraSelector =
|
||||
if (isFrontFacing) CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA
|
||||
|
||||
companion object {
|
||||
private const val TAG = "CameraController"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.example.observer.camera
|
||||
|
||||
import android.os.SystemClock
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.ImageProxy
|
||||
import com.example.observer.detection.DetectionResult
|
||||
import com.example.observer.detection.Detector
|
||||
import com.example.observer.detection.MotionRegion
|
||||
|
||||
class FrameAnalyzer(
|
||||
private val detector: Detector,
|
||||
private val motionDetector: MotionDetector,
|
||||
private val onResult: (results: List<DetectionResult>, rotation: Int, imageWidthPx: Int, imageHeightPx: Int, motionRegions: List<MotionRegion>) -> Unit,
|
||||
) : ImageAnalysis.Analyzer {
|
||||
|
||||
@Volatile var intervalMs: Long = 100L
|
||||
|
||||
private var lastDetectMs = 0L
|
||||
|
||||
override fun analyze(imageProxy: ImageProxy) {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
if (now - lastDetectMs < intervalMs) {
|
||||
imageProxy.close()
|
||||
return
|
||||
}
|
||||
lastDetectMs = now
|
||||
|
||||
val bitmap = imageProxy.toBitmap()
|
||||
val rotation = imageProxy.imageInfo.rotationDegrees
|
||||
try {
|
||||
val motionRegions = motionDetector.detectMotion(bitmap)
|
||||
val results = detector.detect(bitmap)
|
||||
onResult(results, rotation, bitmap.width, bitmap.height, motionRegions)
|
||||
} finally {
|
||||
imageProxy.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.example.observer.camera
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Paint
|
||||
import android.graphics.Rect
|
||||
import com.example.observer.detection.MotionAggregator
|
||||
import com.example.observer.detection.MotionRegion
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
* 轻量运动检测:相邻帧灰度差分 + 分块聚合。
|
||||
* 小尺寸工作(约 128x128 内),每帧开销亚毫秒级,在分析线程串行调用。
|
||||
* 相机大幅移动时(全屏帧差)自动忽略本帧,避免误报。
|
||||
*/
|
||||
class MotionDetector(
|
||||
private val maxWidth: Int = 128,
|
||||
private val maxHeight: Int = 128,
|
||||
) {
|
||||
|
||||
private val smallBitmap = Bitmap.createBitmap(maxWidth, maxHeight, Bitmap.Config.ARGB_8888)
|
||||
private val canvas = Canvas(smallBitmap)
|
||||
private val paint = Paint(Paint.FILTER_BITMAP_FLAG)
|
||||
|
||||
private var prevGray: IntArray? = null
|
||||
private var grayCache: IntArray = IntArray(0)
|
||||
|
||||
fun detectMotion(bitmap: Bitmap): List<MotionRegion> {
|
||||
val scale = min(maxWidth.toFloat() / bitmap.width, maxHeight.toFloat() / bitmap.height)
|
||||
val w = (bitmap.width * scale).toInt()
|
||||
val h = (bitmap.height * scale).toInt()
|
||||
if (w == 0 || h == 0) return emptyList()
|
||||
|
||||
canvas.drawBitmap(bitmap, null, Rect(0, 0, w, h), paint)
|
||||
if (grayCache.size != w * h) grayCache = IntArray(w * h)
|
||||
smallBitmap.getPixels(grayCache, 0, w, 0, 0, w, h)
|
||||
|
||||
val prev = prevGray
|
||||
prevGray = grayCache.copyOf()
|
||||
if (prev == null || prev.size != grayCache.size) return emptyList()
|
||||
|
||||
// 取绿色通道近似亮度
|
||||
val gray = IntArray(grayCache.size)
|
||||
for (i in grayCache.indices) gray[i] = (grayCache[i] shr 8) and 0xFF
|
||||
val prevGreen = IntArray(prev.size)
|
||||
for (i in prev.indices) prevGreen[i] = (prev[i] shr 8) and 0xFF
|
||||
|
||||
val diff = MotionAggregator.diffMask(gray, prevGreen)
|
||||
val motionTotal = diff.sum()
|
||||
// 全屏大差异 → 相机移动/大范围变化,忽略本帧
|
||||
if (motionTotal > w * h / 2) return emptyList()
|
||||
if (motionTotal < 12) return emptyList()
|
||||
return MotionAggregator.aggregate(diff, w, h)
|
||||
}
|
||||
|
||||
/** 相机切换后重置参考帧,避免旧帧误差 */
|
||||
fun reset() {
|
||||
prevGray = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.example.observer.data
|
||||
|
||||
import android.content.Context
|
||||
import androidx.datastore.preferences.core.booleanPreferencesKey
|
||||
import androidx.datastore.preferences.core.edit
|
||||
import androidx.datastore.preferences.core.floatPreferencesKey
|
||||
import androidx.datastore.preferences.core.stringPreferencesKey
|
||||
import androidx.datastore.preferences.preferencesDataStore
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.map
|
||||
|
||||
enum class DetectMode(val intervalMs: Long, val label: String) {
|
||||
CONTINUOUS(100L, "连续"),
|
||||
STANDARD(300L, "标准"),
|
||||
POWER_SAVING(1000L, "省电"),
|
||||
}
|
||||
|
||||
data class AppSettings(
|
||||
val confThreshold: Float = 0.40f,
|
||||
val habitatThreshold: Float = 0.35f,
|
||||
val detectMode: DetectMode = DetectMode.CONTINUOUS,
|
||||
val vibrateEnabled: Boolean = true,
|
||||
val soundEnabled: Boolean = true,
|
||||
val showDistance: Boolean = true,
|
||||
val habitatAlertEnabled: Boolean = true,
|
||||
)
|
||||
|
||||
private val Context.settingsDataStore by preferencesDataStore(name = "settings")
|
||||
|
||||
class SettingsRepository(private val context: Context) {
|
||||
|
||||
val settings: Flow<AppSettings> = context.settingsDataStore.data.map { prefs ->
|
||||
AppSettings(
|
||||
confThreshold = prefs[KEY_CONF] ?: 0.40f,
|
||||
habitatThreshold = prefs[KEY_HABITAT] ?: 0.35f,
|
||||
detectMode = DetectMode.entries.firstOrNull { it.name == prefs[KEY_MODE] }
|
||||
?: DetectMode.CONTINUOUS,
|
||||
vibrateEnabled = prefs[KEY_VIBRATE] ?: true,
|
||||
soundEnabled = prefs[KEY_SOUND] ?: true,
|
||||
showDistance = prefs[KEY_DISTANCE] ?: true,
|
||||
habitatAlertEnabled = prefs[KEY_HABITAT_ALERT] ?: true,
|
||||
)
|
||||
}
|
||||
|
||||
suspend fun setConfThreshold(value: Float) =
|
||||
context.settingsDataStore.edit { it[KEY_CONF] = value }
|
||||
|
||||
suspend fun setHabitatThreshold(value: Float) =
|
||||
context.settingsDataStore.edit { it[KEY_HABITAT] = value }
|
||||
|
||||
suspend fun setDetectMode(mode: DetectMode) =
|
||||
context.settingsDataStore.edit { it[KEY_MODE] = mode.name }
|
||||
|
||||
suspend fun setVibrateEnabled(value: Boolean) =
|
||||
context.settingsDataStore.edit { it[KEY_VIBRATE] = value }
|
||||
|
||||
suspend fun setSoundEnabled(value: Boolean) =
|
||||
context.settingsDataStore.edit { it[KEY_SOUND] = value }
|
||||
|
||||
suspend fun setShowDistance(value: Boolean) =
|
||||
context.settingsDataStore.edit { it[KEY_DISTANCE] = value }
|
||||
|
||||
suspend fun setHabitatAlertEnabled(value: Boolean) =
|
||||
context.settingsDataStore.edit { it[KEY_HABITAT_ALERT] = value }
|
||||
|
||||
companion object {
|
||||
private val KEY_CONF = floatPreferencesKey("conf_threshold")
|
||||
private val KEY_HABITAT = floatPreferencesKey("habitat_threshold")
|
||||
private val KEY_MODE = stringPreferencesKey("detect_mode")
|
||||
private val KEY_VIBRATE = booleanPreferencesKey("vibrate_enabled")
|
||||
private val KEY_SOUND = booleanPreferencesKey("sound_enabled")
|
||||
private val KEY_DISTANCE = booleanPreferencesKey("show_distance")
|
||||
private val KEY_HABITAT_ALERT = booleanPreferencesKey("habitat_alert_enabled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
/** 预览视图像素坐标矩形(纯 JVM 类型,便于单元测试) */
|
||||
data class ViewRect(
|
||||
val left: Float,
|
||||
val top: Float,
|
||||
val right: Float,
|
||||
val bottom: Float,
|
||||
) {
|
||||
val width: Float get() = right - left
|
||||
val height: Float get() = bottom - top
|
||||
val centerX: Float get() = (left + right) / 2f
|
||||
val centerY: Float get() = (top + bottom) / 2f
|
||||
}
|
||||
|
||||
object CoordinateMapper {
|
||||
|
||||
/** 模型归一化坐标 → 预览视图坐标(含传感器旋转与 FIT_CENTER 裁剪) */
|
||||
fun mapToView(
|
||||
normLeft: Float,
|
||||
normTop: Float,
|
||||
normRight: Float,
|
||||
normBottom: Float,
|
||||
rotation: Int, // ImageProxy.imageInfo.rotationDegrees
|
||||
imageW: Int,
|
||||
imageH: Int, // 分析图像尺寸(横屏原图)
|
||||
viewW: Int,
|
||||
viewH: Int, // 预览视图尺寸
|
||||
): ViewRect {
|
||||
// 1) 旋转校正:图像方向 → 竖屏视图方向(归一化坐标)
|
||||
val (x0, y0, x1, y1) = when (rotation) {
|
||||
90 -> Quad(1 - normBottom, normLeft, 1 - normTop, normRight)
|
||||
180 -> Quad(1 - normRight, 1 - normBottom, 1 - normLeft, 1 - normTop)
|
||||
270 -> Quad(normTop, 1 - normRight, normBottom, 1 - normLeft)
|
||||
else -> Quad(normLeft, normTop, normRight, normBottom)
|
||||
}
|
||||
// 2) 旋转后图像在竖屏方向上的尺寸
|
||||
val portrait = rotation == 90 || rotation == 270
|
||||
val portW = if (portrait) imageH else imageW
|
||||
val portH = if (portrait) imageW else imageH
|
||||
// 3) FIT_CENTER 缩放与居中偏移
|
||||
val scale = minOf(viewW.toFloat() / portW, viewH.toFloat() / portH)
|
||||
val offsetX = (viewW - portW * scale) / 2f
|
||||
val offsetY = (viewH - portH * scale) / 2f
|
||||
return ViewRect(
|
||||
left = x0 * portW * scale + offsetX,
|
||||
top = y0 * portH * scale + offsetY,
|
||||
right = x1 * portW * scale + offsetX,
|
||||
bottom = y1 * portH * scale + offsetY,
|
||||
)
|
||||
}
|
||||
|
||||
private data class Quad(val x0: Float, val y0: Float, val x1: Float, val y1: Float)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
data class DetectionResult(
|
||||
val label: String,
|
||||
val score: Float,
|
||||
val left: Float,
|
||||
val top: Float,
|
||||
val right: Float,
|
||||
val bottom: Float,
|
||||
val distanceM: Float? = null,
|
||||
) {
|
||||
val width: Float get() = right - left
|
||||
val height: Float get() = bottom - top
|
||||
val centerX: Float get() = (left + right) / 2f
|
||||
val centerY: Float get() = (top + bottom) / 2f
|
||||
val isHabitat: Boolean get() = label == HABITAT_LABEL
|
||||
|
||||
companion object {
|
||||
const val HABITAT_LABEL = "cover"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import android.graphics.Bitmap
|
||||
|
||||
interface Detector {
|
||||
/** 输入 RGBA 位图,输出归一化坐标检测结果(0..1) */
|
||||
fun detect(bitmap: Bitmap): List<DetectionResult>
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import kotlin.math.abs
|
||||
|
||||
/**
|
||||
* 帧差运动聚合(纯 JVM,可单测)。
|
||||
* 输入:每像素 0/1 差分掩码,按 8x8 分块统计激活块,连通块聚合为运动区域。
|
||||
*/
|
||||
object MotionAggregator {
|
||||
|
||||
private const val BLOCK_GRID = 8
|
||||
private const val BLOCK_ACTIVE_RATIO = 0.30f
|
||||
private const val MAX_REGIONS = 3
|
||||
|
||||
fun aggregate(
|
||||
diff: IntArray,
|
||||
width: Int,
|
||||
height: Int,
|
||||
): List<MotionRegion> {
|
||||
val bw = width / BLOCK_GRID
|
||||
val bh = height / BLOCK_GRID
|
||||
if (bw == 0 || bh == 0) return emptyList()
|
||||
|
||||
val active = BooleanArray(BLOCK_GRID * BLOCK_GRID)
|
||||
for (by in 0 until BLOCK_GRID) {
|
||||
for (bx in 0 until BLOCK_GRID) {
|
||||
val blockW = if (bx == BLOCK_GRID - 1) width - bx * bw else bw
|
||||
val blockH = if (by == BLOCK_GRID - 1) height - by * bh else bh
|
||||
var motion = 0
|
||||
for (y in by * bh until by * bh + blockH) {
|
||||
var idx = y * width + bx * bw
|
||||
for (x in 0 until blockW) {
|
||||
motion += diff[idx + x]
|
||||
}
|
||||
idx += width
|
||||
}
|
||||
active[by * BLOCK_GRID + bx] = motion > blockW * blockH * BLOCK_ACTIVE_RATIO
|
||||
}
|
||||
}
|
||||
|
||||
// 连通块聚合(4 邻域)
|
||||
val regions = ArrayList<MotionRegion>(MAX_REGIONS)
|
||||
val visited = BooleanArray(active.size)
|
||||
for (i in active.indices) {
|
||||
if (!active[i] || visited[i]) continue
|
||||
var minX = BLOCK_GRID
|
||||
var minY = BLOCK_GRID
|
||||
var maxX = -1
|
||||
var maxY = -1
|
||||
val stack = ArrayDeque<Int>()
|
||||
stack.add(i)
|
||||
visited[i] = true
|
||||
while (stack.isNotEmpty()) {
|
||||
val cur = stack.removeLast()
|
||||
val bx = cur % BLOCK_GRID
|
||||
val by = cur / BLOCK_GRID
|
||||
minX = minOf(minX, bx)
|
||||
maxX = maxOf(maxX, bx)
|
||||
minY = minOf(minY, by)
|
||||
maxY = maxOf(maxY, by)
|
||||
for (nb in neighbors(cur)) {
|
||||
if (active[nb] && !visited[nb]) {
|
||||
visited[nb] = true
|
||||
stack.add(nb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (maxX - minX > 3 || maxY - minY > 3) continue // 全屏噪声过滤
|
||||
regions += MotionRegion(
|
||||
left = minX * bw / width.toFloat(),
|
||||
top = minY * bh / height.toFloat(),
|
||||
right = ((maxX + 1) * bw).coerceAtMost(width) / width.toFloat(),
|
||||
bottom = ((maxY + 1) * bh).coerceAtMost(height) / height.toFloat(),
|
||||
)
|
||||
if (regions.size >= MAX_REGIONS) break
|
||||
}
|
||||
return regions
|
||||
}
|
||||
|
||||
private fun neighbors(i: Int): List<Int> {
|
||||
val bx = i % BLOCK_GRID
|
||||
val by = i / BLOCK_GRID
|
||||
val list = ArrayList<Int>(4)
|
||||
if (bx > 0) list += i - 1
|
||||
if (bx < BLOCK_GRID - 1) list += i + 1
|
||||
if (by > 0) list += i - BLOCK_GRID
|
||||
if (by < BLOCK_GRID - 1) list += i + BLOCK_GRID
|
||||
return list
|
||||
}
|
||||
|
||||
/** 检测框中心是否落在运动区域内(用于置信度提升判定) */
|
||||
fun centerInRegion(box: DetectionResult, region: MotionRegion): Boolean =
|
||||
box.centerX in region.left..region.right && box.centerY in region.top..region.bottom
|
||||
|
||||
/** 帧差掩码生成(纯 JVM):|g - prev| > threshold → 1 */
|
||||
fun diffMask(gray: IntArray, prev: IntArray, threshold: Int = 25): IntArray {
|
||||
val diff = IntArray(gray.size)
|
||||
for (i in gray.indices) {
|
||||
diff[i] = if (abs(gray[i] - prev[i]) > threshold) 1 else 0
|
||||
}
|
||||
return diff
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
/** 运动区域(归一化坐标) */
|
||||
data class MotionRegion(
|
||||
val left: Float,
|
||||
val top: Float,
|
||||
val right: Float,
|
||||
val bottom: Float,
|
||||
) {
|
||||
val centerX: Float get() = (left + right) / 2f
|
||||
val centerY: Float get() = (top + bottom) / 2f
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
fun iou(a: DetectionResult, b: DetectionResult): Float {
|
||||
val x0 = maxOf(a.left, b.left)
|
||||
val y0 = maxOf(a.top, b.top)
|
||||
val x1 = minOf(a.right, b.right)
|
||||
val y1 = minOf(a.bottom, b.bottom)
|
||||
if (x1 <= x0 || y1 <= y0) return 0f
|
||||
val inter = (x1 - x0) * (y1 - y0)
|
||||
val union = a.width * a.height + b.width * b.height - inter
|
||||
return if (union <= 0f) 0f else inter / union
|
||||
}
|
||||
|
||||
fun nms(boxes: List<DetectionResult>, iouThreshold: Float): List<DetectionResult> {
|
||||
val sorted = boxes.sortedByDescending { it.score }
|
||||
val kept = mutableListOf<DetectionResult>()
|
||||
for (b in sorted) {
|
||||
if (kept.none { iou(b, it) > iouThreshold }) kept += b
|
||||
}
|
||||
return kept
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Matrix
|
||||
import org.tensorflow.lite.Interpreter
|
||||
import org.tensorflow.lite.gpu.GpuDelegate
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
|
||||
/**
|
||||
* YOLOv8n 端侧推理实现。
|
||||
* 模型输出布局(ultralytics tflite 导出):[1, 4 + nc, 8400],
|
||||
* cx/cy/w/h 已归一化,类别得分已过 sigmoid;按列平铺:index = c * 8400 + anchor。
|
||||
*/
|
||||
class TFLiteDetector private constructor(
|
||||
private val interpreter: Interpreter,
|
||||
private val labels: List<String>,
|
||||
) : Detector {
|
||||
|
||||
@Volatile var confThreshold: Float = 0.40f
|
||||
|
||||
@Volatile var habitatThreshold: Float = 0.35f
|
||||
|
||||
private val scaledBitmap = Bitmap.createBitmap(INPUT_SIZE, INPUT_SIZE, Bitmap.Config.ARGB_8888)
|
||||
private val scaleCanvas = Canvas(scaledBitmap)
|
||||
private val scaleMatrix = Matrix()
|
||||
private val pixels = IntArray(INPUT_SIZE * INPUT_SIZE)
|
||||
|
||||
private val inputBuffer: ByteBuffer =
|
||||
ByteBuffer.allocateDirect(1 * INPUT_SIZE * INPUT_SIZE * 3 * 4)
|
||||
.order(ByteOrder.nativeOrder())
|
||||
|
||||
private val outputFloats = FloatArray((4 + NUM_CLASSES) * NUM_ANCHORS)
|
||||
|
||||
override fun detect(bitmap: Bitmap): List<DetectionResult> {
|
||||
preprocess(bitmap)
|
||||
interpreter.run(inputBuffer, outputFloats)
|
||||
return postprocess()
|
||||
}
|
||||
|
||||
private fun preprocess(bitmap: Bitmap) {
|
||||
// CENTER_CROP 缩放保持宽高比
|
||||
val scale = maxOf(INPUT_SIZE.toFloat() / bitmap.width, INPUT_SIZE.toFloat() / bitmap.height)
|
||||
val dx = (INPUT_SIZE - bitmap.width * scale) / 2f
|
||||
val dy = (INPUT_SIZE - bitmap.height * scale) / 2f
|
||||
scaleMatrix.reset()
|
||||
scaleMatrix.setScale(scale, scale)
|
||||
scaleMatrix.postTranslate(dx, dy)
|
||||
scaleCanvas.drawBitmap(bitmap, scaleMatrix, null)
|
||||
|
||||
inputBuffer.rewind()
|
||||
scaledBitmap.getPixels(pixels, 0, INPUT_SIZE, 0, 0, INPUT_SIZE, INPUT_SIZE)
|
||||
for (p in pixels) {
|
||||
inputBuffer.putFloat(((p shr 16) and 0xFF) / 255f)
|
||||
inputBuffer.putFloat(((p shr 8) and 0xFF) / 255f)
|
||||
inputBuffer.putFloat((p and 0xFF) / 255f)
|
||||
}
|
||||
}
|
||||
|
||||
private fun postprocess(): List<DetectionResult> {
|
||||
val boxes = ArrayList<DetectionResult>(8)
|
||||
for (a in 0 until NUM_ANCHORS) {
|
||||
val cx = outputFloats[a]
|
||||
val cy = outputFloats[NUM_ANCHORS + a]
|
||||
val w = outputFloats[2 * NUM_ANCHORS + a]
|
||||
val h = outputFloats[3 * NUM_ANCHORS + a]
|
||||
var bestCls = 0
|
||||
var bestScore = 0f
|
||||
for (c in 0 until NUM_CLASSES) {
|
||||
val s = outputFloats[(4 + c) * NUM_ANCHORS + a]
|
||||
if (s > bestScore) {
|
||||
bestScore = s
|
||||
bestCls = c
|
||||
}
|
||||
}
|
||||
val label = labels.getOrElse(bestCls) { "unknown" }
|
||||
// cover 用独立阈值;动物类保留低分池,供运动检测提升
|
||||
val threshold = if (label == DetectionResult.HABITAT_LABEL) habitatThreshold else MIN_SCORE
|
||||
if (bestScore < threshold) continue
|
||||
boxes += DetectionResult(
|
||||
label = label,
|
||||
score = bestScore,
|
||||
left = (cx - w / 2f).coerceIn(0f, 1f),
|
||||
top = (cy - h / 2f).coerceIn(0f, 1f),
|
||||
right = (cx + w / 2f).coerceIn(0f, 1f),
|
||||
bottom = (cy + h / 2f).coerceIn(0f, 1f),
|
||||
)
|
||||
}
|
||||
return nms(boxes, IOU_THRESHOLD).take(MAX_DETECTIONS)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val INPUT_SIZE = 320
|
||||
|
||||
/** 动物类最低保留分数:低于此分不输出(低分候选由运动检测提升显示) */
|
||||
const val MIN_SCORE = 0.20f
|
||||
private const val NUM_CLASSES = 4
|
||||
private const val NUM_ANCHORS = 8400
|
||||
private const val IOU_THRESHOLD = 0.45f
|
||||
private const val MAX_DETECTIONS = 20
|
||||
private const val MODEL_ASSET = "model.tflite"
|
||||
private const val LABELS_ASSET = "labels.txt"
|
||||
|
||||
/** 模型缺失或加载失败返回 null(App 降级为仅预览) */
|
||||
fun create(context: Context): TFLiteDetector? {
|
||||
return try {
|
||||
val options = Interpreter.Options().apply {
|
||||
setNumThreads(4)
|
||||
try {
|
||||
addDelegate(GpuDelegate())
|
||||
} catch (_: Throwable) {
|
||||
// GPU 不可用,回退 CPU
|
||||
}
|
||||
}
|
||||
val interpreter = Interpreter(loadModelFile(context), options)
|
||||
val labels = context.assets.open(LABELS_ASSET).bufferedReader().readLines()
|
||||
TFLiteDetector(interpreter, labels)
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun loadModelFile(context: Context): ByteBuffer {
|
||||
val bytes = context.assets.open(MODEL_ASSET).use { it.readBytes() }
|
||||
return ByteBuffer.allocateDirect(bytes.size).order(ByteOrder.nativeOrder())
|
||||
.apply { put(bytes); rewind() }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package com.example.observer.distance
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.camera2.CameraCharacteristics
|
||||
import android.hardware.camera2.CameraManager
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
* 单目距离估计(针孔模型):距离 = 焦距px × 参考体型 / 框高px。
|
||||
* 误差预期 ±30%(5~50m);鱼类目标受折射影响、生境区域按植被高度估算,均仅供参考。
|
||||
*/
|
||||
class DistanceEstimator(context: Context) {
|
||||
|
||||
private val cameraManager =
|
||||
context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
|
||||
|
||||
private val focalPxCache = HashMap<String, Float>()
|
||||
|
||||
// 参考体型(米)
|
||||
private val speciesSizeM = mapOf(
|
||||
"pheasant" to 0.45f, // 身高
|
||||
"hare" to 0.45f, // 身长
|
||||
"dove" to 0.30f, // 体长
|
||||
"fish" to 1.00f, // 典型可见体长(误差大)
|
||||
"cover" to 0.50f, // 植被高度(水面区域误差大)
|
||||
)
|
||||
|
||||
fun estimate(
|
||||
label: String,
|
||||
boxHeightNorm: Float,
|
||||
imageHeightPx: Int,
|
||||
cameraId: String?,
|
||||
): Float? {
|
||||
val realH = speciesSizeM[label] ?: return null
|
||||
val boxH = boxHeightNorm * imageHeightPx
|
||||
if (boxH < 8f) return null // 过小目标不估算
|
||||
val focalPx = focalPx(imageHeightPx, cameraId)
|
||||
if (focalPx <= 0f) return null
|
||||
return (focalPx * realH / boxH).roundToInt().toFloat()
|
||||
}
|
||||
|
||||
/** focal_px = focal_mm × (imageHeightPx / sensorHeightMm) */
|
||||
private fun focalPx(imageHeightPx: Int, cameraId: String?): Float {
|
||||
val key = "$cameraId:$imageHeightPx"
|
||||
focalPxCache[key]?.let { return it }
|
||||
val id = cameraId ?: return -1f
|
||||
val value = try {
|
||||
val c = cameraManager.getCameraCharacteristics(id)
|
||||
val focalMm = c.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)
|
||||
?.firstOrNull()
|
||||
val sensor = c.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)
|
||||
if (focalMm != null && sensor != null) {
|
||||
focalMm * imageHeightPx / sensor.height
|
||||
} else {
|
||||
-1f
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
-1f
|
||||
}
|
||||
focalPxCache[key] = value
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.example.observer.overlay
|
||||
|
||||
import android.graphics.Paint
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.geometry.Size
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.PathEffect
|
||||
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||
import androidx.compose.ui.graphics.nativeCanvas
|
||||
import androidx.compose.ui.graphics.toArgb
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.example.observer.detection.CoordinateMapper
|
||||
import com.example.observer.detection.DetectionResult
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
private val speciesColors = mapOf(
|
||||
"pheasant" to Color(0xFFE53935),
|
||||
"hare" to Color(0xFF1E88E5),
|
||||
"dove" to Color(0xFF8E24AA),
|
||||
"fish" to Color(0xFF00ACC1),
|
||||
"cover" to Color(0xFFFDD835),
|
||||
)
|
||||
|
||||
private val speciesLabels = mapOf(
|
||||
"pheasant" to "野鸡",
|
||||
"hare" to "野兔",
|
||||
"dove" to "斑鸠",
|
||||
"fish" to "鱼",
|
||||
"cover" to "疑似区域",
|
||||
)
|
||||
|
||||
@Composable
|
||||
fun DetectionOverlay(
|
||||
results: List<DetectionResult>,
|
||||
rotation: Int,
|
||||
imageW: Int,
|
||||
imageH: Int,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Canvas(modifier = modifier) {
|
||||
val viewW = size.width.toInt()
|
||||
val viewH = size.height.toInt()
|
||||
val textPaint = Paint().apply {
|
||||
isAntiAlias = true
|
||||
textSize = 30f
|
||||
}
|
||||
results.forEach { r ->
|
||||
val rect = CoordinateMapper.mapToView(
|
||||
r.left, r.top, r.right, r.bottom,
|
||||
rotation, imageW, imageH, viewW, viewH,
|
||||
)
|
||||
val color = speciesColors[r.label] ?: Color.White
|
||||
val stroke = Stroke(
|
||||
width = 6.dp.toPx(),
|
||||
pathEffect = if (r.isHabitat) {
|
||||
PathEffect.dashPathEffect(floatArrayOf(20.dp.toPx(), 12.dp.toPx()))
|
||||
} else {
|
||||
null
|
||||
},
|
||||
)
|
||||
drawRect(
|
||||
color = color,
|
||||
topLeft = Offset(rect.left, rect.top),
|
||||
size = Size(rect.width, rect.height),
|
||||
style = stroke,
|
||||
)
|
||||
textPaint.color = color.toArgb()
|
||||
val text = buildString {
|
||||
append(speciesLabels[r.label] ?: r.label)
|
||||
append(" ${(r.score * 100).toInt()}%")
|
||||
r.distanceM?.let { append(" · 约${it.roundToInt()}m") }
|
||||
}
|
||||
val labelTop = (rect.top - textPaint.textSize - 6).coerceAtLeast(0f)
|
||||
drawContext.canvas.nativeCanvas.drawText(
|
||||
text,
|
||||
rect.left.coerceAtLeast(0f) + 4f,
|
||||
labelTop + textPaint.textSize,
|
||||
textPaint,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package com.example.observer.reminder
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioManager
|
||||
import android.media.ToneGenerator
|
||||
import android.os.Build
|
||||
import android.os.SystemClock
|
||||
import android.os.VibrationEffect
|
||||
import android.os.Vibrator
|
||||
|
||||
class Reminder(context: Context) {
|
||||
|
||||
@Volatile var vibrateEnabled: Boolean = true
|
||||
|
||||
@Volatile var soundEnabled: Boolean = true
|
||||
|
||||
@Volatile var habitatAlertEnabled: Boolean = true
|
||||
|
||||
private val vibrator: Vibrator? =
|
||||
context.getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator
|
||||
|
||||
private val toneGenerator: ToneGenerator? = try {
|
||||
ToneGenerator(AudioManager.STREAM_NOTIFICATION, 60)
|
||||
} catch (e: RuntimeException) {
|
||||
null
|
||||
}
|
||||
|
||||
private var lastAlertLabel: String? = null
|
||||
private var lastAlertAt = 0L
|
||||
|
||||
/** 同类目标 10s 内只提醒一次;生境区域(cover)提醒方式与动物检测区分 */
|
||||
fun onDetected(label: String) {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
if (label == lastAlertLabel && now - lastAlertAt < 10_000) return
|
||||
lastAlertAt = now
|
||||
lastAlertLabel = label
|
||||
|
||||
val isHabitat = label == "cover"
|
||||
if (isHabitat && !habitatAlertEnabled) return
|
||||
|
||||
if (vibrateEnabled) vibrate(isHabitat)
|
||||
if (soundEnabled) playTone(isHabitat)
|
||||
}
|
||||
|
||||
private fun vibrate(isHabitat: Boolean) {
|
||||
val v = vibrator ?: return
|
||||
if (Build.VERSION.SDK_INT >= 26) {
|
||||
val effect = if (isHabitat) {
|
||||
VibrationEffect.createWaveform(longArrayOf(0, 150, 80, 150), -1)
|
||||
} else {
|
||||
VibrationEffect.createOneShot(200, VibrationEffect.DEFAULT_AMPLITUDE)
|
||||
}
|
||||
v.vibrate(effect)
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
v.vibrate(if (isHabitat) longArrayOf(0, 150, 80, 150) else longArrayOf(0, 200), -1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun playTone(isHabitat: Boolean) {
|
||||
val tone = toneGenerator ?: return
|
||||
if (isHabitat) {
|
||||
// 双短音:生境预警
|
||||
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 120)
|
||||
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 120)
|
||||
} else {
|
||||
tone.startTone(ToneGenerator.TONE_PROP_BEEP2, 200)
|
||||
}
|
||||
}
|
||||
|
||||
fun release() {
|
||||
toneGenerator?.release()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.example.observer.ui
|
||||
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import com.example.observer.ui.camera.CameraScreen
|
||||
import com.example.observer.ui.settings.SettingsScreen
|
||||
import com.example.observer.ui.theme.ObserverTheme
|
||||
|
||||
@Composable
|
||||
fun ObserverApp() {
|
||||
ObserverTheme {
|
||||
var showSettings by rememberSaveable { mutableStateOf(false) }
|
||||
if (showSettings) {
|
||||
SettingsScreen(onBack = { showSettings = false })
|
||||
} else {
|
||||
CameraScreen(onOpenSettings = { showSettings = true })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package com.example.observer.ui.camera
|
||||
|
||||
import android.Manifest
|
||||
import android.content.pm.PackageManager
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableIntStateOf
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalLifecycleOwner
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import androidx.lifecycle.viewmodel.compose.viewModel
|
||||
import com.example.observer.ObserverApp
|
||||
import com.example.observer.camera.CameraController
|
||||
import com.example.observer.camera.FrameAnalyzer
|
||||
import com.example.observer.camera.MotionDetector
|
||||
import com.example.observer.data.AppSettings
|
||||
import com.example.observer.data.DetectMode
|
||||
import com.example.observer.overlay.DetectionOverlay
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
@Composable
|
||||
fun CameraScreen(onOpenSettings: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val container = (context.applicationContext as ObserverApp).container
|
||||
val viewModel: CameraViewModel = viewModel(
|
||||
factory = remember { CameraViewModelFactory(container) },
|
||||
)
|
||||
val state by viewModel.state.collectAsStateWithLifecycle()
|
||||
val settings by container.settingsRepository.settings.collectAsStateWithLifecycle(
|
||||
initialValue = AppSettings(),
|
||||
)
|
||||
val scope = rememberCoroutineScope()
|
||||
|
||||
val executor = remember { Executors.newSingleThreadExecutor() }
|
||||
val previewView = remember {
|
||||
PreviewView(context).apply {
|
||||
scaleType = PreviewView.ScaleType.FIT_CENTER
|
||||
implementationMode = PreviewView.ImplementationMode.PERFORMANCE
|
||||
}
|
||||
}
|
||||
val cameraController = remember { CameraController(lifecycleOwner, executor) }
|
||||
val motionDetector = remember { MotionDetector() }
|
||||
val analyzer = remember(container.detector) {
|
||||
container.detector?.let { d ->
|
||||
FrameAnalyzer(d, motionDetector) { results, rotation, w, h, motion ->
|
||||
viewModel.onFramesAnalyzed(results, rotation, w, h, motion)
|
||||
}
|
||||
}
|
||||
}
|
||||
val interval by viewModel.analyzerIntervalMs.collectAsState()
|
||||
|
||||
var hasCameraPermission by remember {
|
||||
mutableStateOf(
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
|
||||
PackageManager.PERMISSION_GRANTED,
|
||||
)
|
||||
}
|
||||
val permissionLauncher = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.RequestPermission(),
|
||||
) { granted -> hasCameraPermission = granted }
|
||||
var retryKey by remember { mutableIntStateOf(0) }
|
||||
var initFailed by remember { mutableStateOf(false) }
|
||||
|
||||
DisposableEffect(Unit) {
|
||||
cameraController.onCameraIdChanged = { viewModel.onCameraIdChanged(it) }
|
||||
onDispose { executor.shutdown() }
|
||||
}
|
||||
LaunchedEffect(interval) { analyzer?.intervalMs = interval }
|
||||
LaunchedEffect(retryKey) {
|
||||
cameraController.onInitError = { initFailed = true }
|
||||
if (hasCameraPermission) cameraController.start(previewView, analyzer)
|
||||
}
|
||||
LaunchedEffect(Unit) {
|
||||
if (!hasCameraPermission) permissionLauncher.launch(Manifest.permission.CAMERA)
|
||||
}
|
||||
|
||||
Box(modifier = Modifier.fillMaxSize().background(Color.Black)) {
|
||||
if (hasCameraPermission) {
|
||||
AndroidView(factory = { previewView }, modifier = Modifier.fillMaxSize())
|
||||
|
||||
if (state.modelReady) {
|
||||
DetectionOverlay(
|
||||
results = state.results,
|
||||
rotation = state.rotation,
|
||||
imageW = state.imageWidthPx,
|
||||
imageH = state.imageHeightPx,
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
)
|
||||
} else {
|
||||
Box(
|
||||
modifier = Modifier.fillMaxSize().padding(24.dp),
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
Text(
|
||||
"模型未加载:请将训练好的 model.tflite 放入 app/src/main/assets 后重新构建",
|
||||
color = Color.White,
|
||||
textAlign = TextAlign.Center,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (initFailed) {
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.align(Alignment.Center)
|
||||
.background(Color(0x99000000), RoundedCornerShape(12.dp))
|
||||
.padding(24.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
) {
|
||||
Text("相机初始化失败", color = Color.White)
|
||||
Spacer(Modifier.height(8.dp))
|
||||
TextButton(onClick = {
|
||||
initFailed = false
|
||||
retryKey++
|
||||
}) { Text("重试") }
|
||||
}
|
||||
}
|
||||
|
||||
CameraTopBar(
|
||||
detectMode = settings.detectMode,
|
||||
onCycleMode = {
|
||||
val next = DetectMode.entries[
|
||||
(settings.detectMode.ordinal + 1) % DetectMode.entries.size
|
||||
]
|
||||
scope.launch { container.settingsRepository.setDetectMode(next) }
|
||||
},
|
||||
onSwitchCamera = { cameraController.switchCamera(previewView, analyzer) },
|
||||
modifier = Modifier.align(Alignment.TopCenter),
|
||||
)
|
||||
CameraBottomBar(
|
||||
vibrateEnabled = settings.vibrateEnabled,
|
||||
onToggleVibrate = {
|
||||
scope.launch {
|
||||
container.settingsRepository.setVibrateEnabled(!settings.vibrateEnabled)
|
||||
}
|
||||
},
|
||||
onOpenSettings = onOpenSettings,
|
||||
modifier = Modifier.align(Alignment.BottomCenter),
|
||||
)
|
||||
} else {
|
||||
PermissionGuide(
|
||||
onRequest = { permissionLauncher.launch(Manifest.permission.CAMERA) },
|
||||
modifier = Modifier.align(Alignment.Center),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun CameraTopBar(
|
||||
detectMode: DetectMode,
|
||||
onCycleMode: () -> Unit,
|
||||
onSwitchCamera: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Row(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.background(Color(0x99000000))
|
||||
.padding(horizontal = 12.dp, vertical = 4.dp),
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
TextButton(onClick = onCycleMode) {
|
||||
Text("频率:${detectMode.label}", color = Color.White)
|
||||
}
|
||||
TextButton(onClick = onSwitchCamera) {
|
||||
Text("切换摄像头", color = Color.White)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun CameraBottomBar(
|
||||
vibrateEnabled: Boolean,
|
||||
onToggleVibrate: () -> Unit,
|
||||
onOpenSettings: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Row(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.background(Color(0x99000000))
|
||||
.padding(horizontal = 12.dp, vertical = 4.dp),
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
TextButton(onClick = onToggleVibrate) {
|
||||
Text(if (vibrateEnabled) "提醒:开" else "提醒:关", color = Color.White)
|
||||
}
|
||||
TextButton(onClick = onOpenSettings) {
|
||||
Text("设置", color = Color.White)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun PermissionGuide(onRequest: () -> Unit, modifier: Modifier = Modifier) {
|
||||
Column(
|
||||
modifier = modifier.padding(32.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
) {
|
||||
Text(
|
||||
"需要相机权限才能进行实时识别",
|
||||
color = Color.White,
|
||||
textAlign = TextAlign.Center,
|
||||
)
|
||||
Spacer(Modifier.height(16.dp))
|
||||
Button(onClick = onRequest) { Text("授权相机") }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package com.example.observer.ui.camera
|
||||
|
||||
import android.os.SystemClock
|
||||
import androidx.lifecycle.ViewModel
|
||||
import androidx.lifecycle.ViewModelProvider
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import com.example.observer.AppContainer
|
||||
import com.example.observer.data.DetectMode
|
||||
import com.example.observer.data.SettingsRepository
|
||||
import com.example.observer.detection.DetectionResult
|
||||
import com.example.observer.detection.MotionAggregator
|
||||
import com.example.observer.detection.MotionRegion
|
||||
import com.example.observer.detection.TFLiteDetector
|
||||
import com.example.observer.distance.DistanceEstimator
|
||||
import com.example.observer.reminder.Reminder
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.flow.update
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.LinkedHashMap
|
||||
|
||||
data class CameraUiState(
|
||||
val modelReady: Boolean = false,
|
||||
val results: List<DetectionResult> = emptyList(),
|
||||
val rotation: Int = 90,
|
||||
val imageWidthPx: Int = 0,
|
||||
val imageHeightPx: Int = 0,
|
||||
)
|
||||
|
||||
class CameraViewModel(
|
||||
private val detector: TFLiteDetector?,
|
||||
private val distanceEstimator: DistanceEstimator,
|
||||
private val reminder: Reminder,
|
||||
settingsRepository: SettingsRepository,
|
||||
) : ViewModel() {
|
||||
|
||||
private val _state = MutableStateFlow(CameraUiState(modelReady = detector != null))
|
||||
val state: StateFlow<CameraUiState> = _state.asStateFlow()
|
||||
|
||||
private val _analyzerIntervalMs = MutableStateFlow(DetectMode.CONTINUOUS.intervalMs)
|
||||
val analyzerIntervalMs: StateFlow<Long> = _analyzerIntervalMs.asStateFlow()
|
||||
|
||||
@Volatile
|
||||
private var showDistance = true
|
||||
|
||||
@Volatile
|
||||
private var confThreshold = 0.40f
|
||||
|
||||
@Volatile
|
||||
private var cameraId: String? = null
|
||||
|
||||
private val tracks = LinkedHashMap<String, Track>()
|
||||
|
||||
init {
|
||||
viewModelScope.launch {
|
||||
settingsRepository.settings.collect { s ->
|
||||
detector?.confThreshold = s.confThreshold
|
||||
detector?.habitatThreshold = s.habitatThreshold
|
||||
reminder.vibrateEnabled = s.vibrateEnabled
|
||||
reminder.soundEnabled = s.soundEnabled
|
||||
reminder.habitatAlertEnabled = s.habitatAlertEnabled
|
||||
showDistance = s.showDistance
|
||||
confThreshold = s.confThreshold
|
||||
_analyzerIntervalMs.value = s.detectMode.intervalMs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun onCameraIdChanged(id: String) {
|
||||
cameraId = id
|
||||
}
|
||||
|
||||
/** 帧分析回调(分析线程调用,需线程安全) */
|
||||
fun onFramesAnalyzed(
|
||||
results: List<DetectionResult>,
|
||||
rotation: Int,
|
||||
imageWidthPx: Int,
|
||||
imageHeightPx: Int,
|
||||
motionRegions: List<MotionRegion>,
|
||||
) {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
val byKey = HashMap<String, DetectionResult>(results.size)
|
||||
results.forEach { byKey[keyOf(it)] = it }
|
||||
|
||||
synchronized(this) {
|
||||
// 过期目标清除(消失 2s 后移除)
|
||||
val expired = tracks.entries
|
||||
.filter { it.key !in byKey && now - it.value.lastSeenMs > 2000 }
|
||||
.map { it.key }
|
||||
expired.forEach { tracks.remove(it) }
|
||||
|
||||
byKey.forEach { (k, r) ->
|
||||
val t = tracks[k]
|
||||
tracks[k] = if (t == null) Track(now, now, r) else Track(t.firstSeenMs, now, r)
|
||||
}
|
||||
if (tracks.size > MAX_TRACKS) {
|
||||
val oldest = tracks.entries.sortedBy { it.value.lastSeenMs }.take(tracks.size - MAX_TRACKS)
|
||||
oldest.forEach { tracks.remove(it.key) }
|
||||
}
|
||||
|
||||
// 目标停留 ≥ 0.5s 才展示;消失 2s 内不闪断
|
||||
val visible = tracks.values
|
||||
.filter { now - it.firstSeenMs >= 500 && now - it.lastSeenMs <= 2000 }
|
||||
.filter { shouldShow(it.result, motionRegions) }
|
||||
.map { t ->
|
||||
val r = t.result
|
||||
val boosted = r.score < confThreshold &&
|
||||
motionRegions.any { MotionAggregator.centerInRegion(r, it) }
|
||||
val distance = if (showDistance) {
|
||||
distanceEstimator.estimate(r.label, r.height, imageHeightPx, cameraId)
|
||||
} else {
|
||||
null
|
||||
}
|
||||
r.copy(
|
||||
score = if (boosted) minOf(r.score + MOTION_BOOST, 1f) else r.score,
|
||||
distanceM = distance,
|
||||
)
|
||||
}
|
||||
|
||||
// 提醒:刚变为可见的目标(防重复由 Reminder 按类别 10s 控制)
|
||||
tracks.values.forEach { t ->
|
||||
if (!shouldShow(t.result, motionRegions)) return@forEach
|
||||
val age = now - t.firstSeenMs
|
||||
if (age in 500..2100 && now - t.lastSeenMs <= 300) {
|
||||
reminder.onDetected(t.result.label)
|
||||
}
|
||||
}
|
||||
|
||||
_state.update {
|
||||
it.copy(
|
||||
results = visible,
|
||||
rotation = rotation,
|
||||
imageWidthPx = imageWidthPx,
|
||||
imageHeightPx = imageHeightPx,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示判定:cover 已按生境阈值过滤;动物分低于阈值时,
|
||||
* 仅当与运动区域重叠(低分候选)才提升显示。
|
||||
*/
|
||||
private fun shouldShow(r: DetectionResult, motionRegions: List<MotionRegion>): Boolean {
|
||||
if (r.isHabitat) return true
|
||||
if (r.score >= confThreshold) return true
|
||||
if (r.score < TFLiteDetector.MIN_SCORE) return false
|
||||
return motionRegions.any { MotionAggregator.centerInRegion(r, it) }
|
||||
}
|
||||
|
||||
override fun onCleared() {
|
||||
reminder.release()
|
||||
}
|
||||
|
||||
private fun keyOf(r: DetectionResult): String =
|
||||
"${r.label}:${(r.centerX * 10).toInt()}:${(r.centerY * 10).toInt()}"
|
||||
|
||||
private data class Track(
|
||||
val firstSeenMs: Long,
|
||||
val lastSeenMs: Long,
|
||||
val result: DetectionResult,
|
||||
)
|
||||
|
||||
companion object {
|
||||
private const val MAX_TRACKS = 30
|
||||
|
||||
/** 低分目标与运动区域重叠时的置信度提升量 */
|
||||
private const val MOTION_BOOST = 0.15f
|
||||
}
|
||||
}
|
||||
|
||||
class CameraViewModelFactory(private val container: AppContainer) : ViewModelProvider.Factory {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
override fun <T : ViewModel> create(modelClass: Class<T>): T {
|
||||
if (modelClass.isAssignableFrom(CameraViewModel::class.java)) {
|
||||
return CameraViewModel(
|
||||
detector = container.detector as? TFLiteDetector,
|
||||
distanceEstimator = container.distanceEstimator,
|
||||
reminder = container.reminder,
|
||||
settingsRepository = container.settingsRepository,
|
||||
) as T
|
||||
}
|
||||
throw IllegalArgumentException("Unknown ViewModel class: ${modelClass.name}")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package com.example.observer.ui.settings
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.FilterChip
|
||||
import androidx.compose.material3.Scaffold
|
||||
import androidx.compose.material3.Slider
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.material3.TopAppBar
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.example.observer.ObserverApp
|
||||
import com.example.observer.data.AppSettings
|
||||
import com.example.observer.data.DetectMode
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
fun SettingsScreen(onBack: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val repository = (context.applicationContext as ObserverApp).container.settingsRepository
|
||||
val settings by repository.settings.collectAsStateWithLifecycle(initialValue = AppSettings())
|
||||
val scope = rememberCoroutineScope()
|
||||
|
||||
Scaffold(
|
||||
topBar = {
|
||||
TopAppBar(
|
||||
title = { Text("设置") },
|
||||
navigationIcon = {
|
||||
TextButton(onClick = onBack) { Text("返回") }
|
||||
},
|
||||
)
|
||||
},
|
||||
) { padding ->
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.padding(padding)
|
||||
.fillMaxSize()
|
||||
.verticalScroll(rememberScrollState())
|
||||
.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text("动物识别阈值:${"%.2f".format(settings.confThreshold)}")
|
||||
Slider(
|
||||
value = settings.confThreshold,
|
||||
onValueChange = { scope.launch { repository.setConfThreshold(it) } },
|
||||
valueRange = 0.2f..0.7f,
|
||||
)
|
||||
|
||||
Text("生境区域阈值:${"%.2f".format(settings.habitatThreshold)}")
|
||||
Slider(
|
||||
value = settings.habitatThreshold,
|
||||
onValueChange = { scope.launch { repository.setHabitatThreshold(it) } },
|
||||
valueRange = 0.2f..0.7f,
|
||||
)
|
||||
|
||||
Text("检测频率")
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
DetectMode.entries.forEach { mode ->
|
||||
FilterChip(
|
||||
selected = settings.detectMode == mode,
|
||||
onClick = { scope.launch { repository.setDetectMode(mode) } },
|
||||
label = { Text(mode.label) },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
SwitchRow("震动提醒", settings.vibrateEnabled) {
|
||||
scope.launch { repository.setVibrateEnabled(it) }
|
||||
}
|
||||
SwitchRow("提示音", settings.soundEnabled) {
|
||||
scope.launch { repository.setSoundEnabled(it) }
|
||||
}
|
||||
SwitchRow("距离标注", settings.showDistance) {
|
||||
scope.launch { repository.setShowDistance(it) }
|
||||
}
|
||||
SwitchRow("生境区域预警", settings.habitatAlertEnabled) {
|
||||
scope.launch { repository.setHabitatAlertEnabled(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SwitchRow(title: String, checked: Boolean, onCheckedChange: (Boolean) -> Unit) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Text(title)
|
||||
Switch(checked = checked, onCheckedChange = onCheckedChange)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.example.observer.ui.theme
|
||||
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.darkColorScheme
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.graphics.Color
|
||||
|
||||
private val DarkColors = darkColorScheme(
|
||||
primary = Color(0xFF66BB6A),
|
||||
onPrimary = Color(0xFF00391F),
|
||||
secondary = Color(0xFF80CBC4),
|
||||
surface = Color(0xFF1C1C1C),
|
||||
background = Color(0xFF101010),
|
||||
onBackground = Color(0xFFE4E4E4),
|
||||
onSurface = Color(0xFFE4E4E4),
|
||||
)
|
||||
|
||||
@Composable
|
||||
fun ObserverTheme(content: @Composable () -> Unit) {
|
||||
MaterialTheme(colorScheme = DarkColors, content = content)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="108dp"
|
||||
android:height="108dp"
|
||||
android:viewportWidth="108"
|
||||
android:viewportHeight="108">
|
||||
<path
|
||||
android:fillColor="#1B5E20"
|
||||
android:pathData="M54,54m-54,0a54,54 0,1 1,108 0a54,54 0,1 1,-108 0" />
|
||||
<path
|
||||
android:fillColor="#00000000"
|
||||
android:strokeColor="#FFFFFF"
|
||||
android:strokeWidth="6"
|
||||
android:pathData="M54,54m-20,0a20,20 0,1 1,40 0a20,20 0,1 1,-40 0" />
|
||||
<path
|
||||
android:strokeColor="#FFFFFF"
|
||||
android:strokeWidth="4"
|
||||
android:pathData="M54,22 L54,34 M54,74 L54,86 M22,54 L34,54 M74,54 L86,54" />
|
||||
<path
|
||||
android:fillColor="#FFFFFF"
|
||||
android:pathData="M54,54m-5,0a5,5 0,1 1,10 0a5,5 0,1 1,-10 0" />
|
||||
</vector>
|
||||
@@ -0,0 +1,3 @@
|
||||
<resources>
|
||||
<string name="app_name">野视</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,7 @@
|
||||
<resources>
|
||||
<style name="Theme.Observer" parent="android:Theme.Material.Light.NoActionBar">
|
||||
<item name="android:windowBackground">@android:color/black</item>
|
||||
<item name="android:statusBarColor">@android:color/transparent</item>
|
||||
<item name="android:navigationBarColor">@android:color/black</item>
|
||||
</style>
|
||||
</resources>
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
class CoordinateMapperTest {
|
||||
|
||||
@Test
|
||||
fun rotation90_center_mapsToViewCenter() {
|
||||
// 横屏图像 1280x720,旋转 90 后竖屏显示 720x1280,视图 1080x1920(等比,无偏移)
|
||||
val rect = CoordinateMapper.mapToView(
|
||||
normLeft = 0.4f, normTop = 0.4f, normRight = 0.6f, normBottom = 0.6f,
|
||||
rotation = 90, imageW = 1280, imageH = 720,
|
||||
viewW = 1080, viewH = 1920,
|
||||
)
|
||||
assertEquals(540f, rect.centerX, 1f)
|
||||
assertEquals(960f, rect.centerY, 1f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rotation90_corner_appliesFitCenterOffset() {
|
||||
// 视图 1080x2400,竖屏图像 720x1280,scale=1.5,纵向偏移 (2400-1920)/2=240
|
||||
val rect = CoordinateMapper.mapToView(
|
||||
normLeft = 0f, normTop = 0f, normRight = 0.5f, normBottom = 0.5f,
|
||||
rotation = 90, imageW = 1280, imageH = 720,
|
||||
viewW = 1080, viewH = 2400,
|
||||
)
|
||||
// 旋转后:x0=1-bottom=0.5, y0=left=0, x1=1-top=1, y1=right=0.5
|
||||
assertEquals(540f, rect.left, 1f) // 0.5 * 720 * 1.5
|
||||
assertEquals(240f, rect.top, 1f) // 0 * 1280 * 1.5 + 240
|
||||
assertEquals(1080f, rect.right, 1f) // 1 * 720 * 1.5
|
||||
assertEquals(1200f, rect.bottom, 1f) // 0.5 * 1280 * 1.5 + 240
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rotation0_keepsCoordinates() {
|
||||
val rect = CoordinateMapper.mapToView(
|
||||
normLeft = 0.2f, normTop = 0.3f, normRight = 0.5f, normBottom = 0.7f,
|
||||
rotation = 0, imageW = 1080, imageH = 1920,
|
||||
viewW = 1080, viewH = 1920,
|
||||
)
|
||||
assertEquals(216f, rect.left, 1f)
|
||||
assertEquals(576f, rect.top, 1f)
|
||||
assertEquals(540f, rect.right, 1f)
|
||||
assertEquals(1344f, rect.bottom, 1f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rotation180_flipsBothAxes() {
|
||||
val rect = CoordinateMapper.mapToView(
|
||||
normLeft = 0.2f, normTop = 0.2f, normRight = 0.4f, normBottom = 0.4f,
|
||||
rotation = 180, imageW = 1080, imageH = 1920,
|
||||
viewW = 1080, viewH = 1920,
|
||||
)
|
||||
// 翻转:x'=1-x, y'=1-y
|
||||
assertEquals(648f, rect.left, 1f) // (1-0.4) * 1080
|
||||
assertEquals(1152f, rect.top, 1f) // (1-0.4) * 1920
|
||||
assertEquals(864f, rect.right, 1f) // (1-0.2) * 1080
|
||||
assertEquals(1536f, rect.bottom, 1f) // (1-0.2) * 1920
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class MotionAggregatorTest {
|
||||
|
||||
// 96x64 图,8x8 块 → 每块 12x8 像素
|
||||
|
||||
@Test
|
||||
fun noMotion_returnsEmpty() {
|
||||
val diff = IntArray(96 * 64)
|
||||
assertTrue(MotionAggregator.aggregate(diff, 96, 64).isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun singleBlockMotion_detectsRegion() {
|
||||
val diff = IntArray(96 * 64)
|
||||
// 块 (2,3):x 24..35, y 24..31,全部置 1
|
||||
for (y in 24 until 32) {
|
||||
for (x in 24 until 36) diff[y * 96 + x] = 1
|
||||
}
|
||||
val regions = MotionAggregator.aggregate(diff, 96, 64)
|
||||
assertEquals(1, regions.size)
|
||||
val r = regions[0]
|
||||
assertTrue(r.left <= 24f / 96f && r.right >= 36f / 96f)
|
||||
assertTrue(r.top <= 24f / 64f && r.bottom >= 32f / 64f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun twoSeparateMotions_detectsTwoRegions() {
|
||||
val diff = IntArray(96 * 64)
|
||||
for (y in 0 until 8) for (x in 0 until 12) diff[y * 96 + x] = 1
|
||||
for (y in 48 until 64) for (x in 72 until 96) diff[y * 96 + x] = 1
|
||||
val regions = MotionAggregator.aggregate(diff, 96, 64)
|
||||
assertEquals(2, regions.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun globalNoise_filteredOut() {
|
||||
val diff = IntArray(96 * 64)
|
||||
// 全屏散点噪声(随机块),但无连续连通域
|
||||
val rnd = java.util.Random(42)
|
||||
for (i in diff.indices) if (rnd.nextFloat() < 0.1f) diff[i] = 1
|
||||
val regions = MotionAggregator.aggregate(diff, 96, 64)
|
||||
assertTrue(regions.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun centerInRegion_matches() {
|
||||
val box = DetectionResult(
|
||||
label = "hare",
|
||||
score = 0.30f,
|
||||
left = 0.2f,
|
||||
top = 0.3f,
|
||||
right = 0.4f,
|
||||
bottom = 0.5f,
|
||||
)
|
||||
val region = MotionRegion(0.1f, 0.2f, 0.5f, 0.6f)
|
||||
assertTrue(MotionAggregator.centerInRegion(box, region))
|
||||
val outside = MotionRegion(0.6f, 0.7f, 0.9f, 0.9f)
|
||||
assertFalse(MotionAggregator.centerInRegion(box, outside))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.example.observer.detection
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class NmsTest {
|
||||
|
||||
private fun box(l: Float, t: Float, r: Float, b: Float, score: Float, label: String = "x") =
|
||||
DetectionResult(label, score, l, t, r, b)
|
||||
|
||||
@Test
|
||||
fun overlappingBoxes_keepHighestScore() {
|
||||
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 0.8f)
|
||||
val b = box(0.12f, 0.12f, 0.52f, 0.52f, 0.6f)
|
||||
val result = nms(listOf(a, b), 0.45f)
|
||||
assertEquals(1, result.size)
|
||||
assertEquals(0.8f, result[0].score, 1e-6f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun separateBoxes_bothKept() {
|
||||
val a = box(0.1f, 0.1f, 0.3f, 0.3f, 0.8f)
|
||||
val b = box(0.7f, 0.7f, 0.9f, 0.9f, 0.6f)
|
||||
val result = nms(listOf(a, b), 0.45f)
|
||||
assertEquals(2, result.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lowScoreBox_suppressedByHigherScore() {
|
||||
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 0.9f)
|
||||
val b = box(0.1f, 0.1f, 0.5f, 0.5f, 0.5f)
|
||||
val result = nms(listOf(b, a), 0.45f)
|
||||
assertEquals(1, result.size)
|
||||
assertTrue(result[0].score > 0.5f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun iou_nonOverlapping_isZero() {
|
||||
val a = box(0.0f, 0.0f, 0.2f, 0.2f, 1f)
|
||||
val b = box(0.8f, 0.8f, 1f, 1f, 1f)
|
||||
assertEquals(0f, iou(a, b), 1e-6f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun iou_identicalBoxes_isOne() {
|
||||
val a = box(0.1f, 0.1f, 0.5f, 0.5f, 1f)
|
||||
assertEquals(1f, iou(a, a), 1e-6f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
plugins {
|
||||
alias(libs.plugins.android.application) apply false
|
||||
alias(libs.plugins.kotlin.android) apply false
|
||||
alias(libs.plugins.kotlin.compose) apply false
|
||||
}
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
# Observer(野视)· 野生动物实时识别 Android App — 技术方案
|
||||
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 文档版本 | v1.1 |
|
||||
| 编写日期 | 2026-08-17 |
|
||||
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照留存) |
|
||||
| 适用产品 | 纯 Android 原生 App |
|
||||
|
||||
---
|
||||
|
||||
## 1. 项目概述
|
||||
|
||||
### 1.1 项目背景
|
||||
|
||||
野生动物观察爱好者在户外需要一款便携工具:打开手机相机,即可从实时画面中识别野鸡,通过检测框与提醒辅助快速发现。
|
||||
|
||||
### 1.2 项目目标
|
||||
|
||||
- 端侧实时目标检测,**全程离线可用**,不依赖网络;
|
||||
- 野鸡实时框选识别,展示类别、置信度与**大致距离**;
|
||||
- 检测到目标时通过震动 / 声音提醒用户,辅助快速发现;
|
||||
- 纯 Android 原生 App(Kotlin),兼容 Android 7.0+ 主流机型,中端机流畅运行;
|
||||
- **不做拍照、记录、统计等留存功能**,专注实时识别这一核心体验。
|
||||
|
||||
### 1.3 名词术语
|
||||
|
||||
| 术语 | 说明 |
|
||||
| --- | --- |
|
||||
| ImageAnalysis | CameraX 的帧分析用例,用于逐帧回调图像数据 |
|
||||
| TFLite | TensorFlow Lite,Google 端侧推理框架 |
|
||||
| YOLO | You Only Look Once,单阶段目标检测算法 |
|
||||
| NMS | Non-Maximum Suppression,非极大值抑制 |
|
||||
| GPU Delegate | TFLite 的 GPU 加速委托,将算子下发 GPU 执行 |
|
||||
| mAP | mean Average Precision,目标检测平均精度指标 |
|
||||
| IoU | Intersection over Union,交并比 |
|
||||
|
||||
---
|
||||
|
||||
## 2. 需求概述
|
||||
|
||||
### 2.1 目标用户
|
||||
|
||||
| 用户群 | 典型诉求 |
|
||||
| --- | --- |
|
||||
| 野生动物观察爱好者 | 户外实时识别画面中的野鸡,不惊扰、近距离观察 |
|
||||
| 户外徒步 / 摄影人群 | 快速发现野鸡,辅助取景构图 |
|
||||
| 自然教育、科普工作者 | 物种识别辅助教学 |
|
||||
|
||||
### 2.2 核心场景
|
||||
|
||||
| 编号 | 场景 | 描述 |
|
||||
| --- | --- | --- |
|
||||
| S1 | 野外实时识别 | 徒步 / 观鸟时打开相机,实时框选画面中的野鸡,显示类别、置信度与大致距离 |
|
||||
| S2 | 快速扫视寻找 | 移动取景快速扫视,检测到野鸡立即震动 / 声音提醒,无需停留操作 |
|
||||
|
||||
### 2.3 功能需求摘要
|
||||
|
||||
实时识别(P0)、识别结果叠加展示(P0)、检测提醒(P1)、生境区域预警(P1)、设置(P1)、合规提示(P0)、低光增强(P2)。详见《项目功能文档》。
|
||||
|
||||
### 2.4 非功能需求摘要
|
||||
|
||||
| 指标 | 目标 |
|
||||
| --- | --- |
|
||||
| 识别延迟 | 中端机单帧 ≤ 80ms |
|
||||
| 预览流畅度 | 连续模式 2~3 帧检测一次,预览不卡顿 |
|
||||
| 耗电 | 连续使用 1 小时耗电 ≤ 15% |
|
||||
| 兼容性 | Android 7.0+(minSdk 24),覆盖主流国产机与三星 |
|
||||
| 稳定性 | 崩溃率 ≤ 0.5%,启动成功率 ≥ 99% |
|
||||
|
||||
---
|
||||
|
||||
## 3. 总体架构
|
||||
|
||||
### 3.1 架构图
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph UI层["UI 层 · Jetpack Compose"]
|
||||
MainScreen["相机主界面<br/>预览 + 检测叠加层"]
|
||||
SettingsScreen["设置界面"]
|
||||
end
|
||||
|
||||
subgraph ViewModel层["ViewModel 层"]
|
||||
CameraViewModel["CameraViewModel"]
|
||||
SettingsViewModel["SettingsViewModel"]
|
||||
end
|
||||
|
||||
subgraph 领域层["领域层"]
|
||||
Detector接口["Detector 接口"]
|
||||
end
|
||||
|
||||
subgraph 基础设施层["基础设施层"]
|
||||
CameraX["CameraX<br/>Preview / ImageAnalysis"]
|
||||
TFLite["TFLite 推理引擎<br/>YOLOv8n 模型 + GPU Delegate"]
|
||||
DataStore["DataStore 设置存储"]
|
||||
end
|
||||
|
||||
UI层 --> ViewModel层
|
||||
ViewModel层 --> 领域层
|
||||
领域层 --> 基础设施层
|
||||
```
|
||||
|
||||
### 3.2 分层说明
|
||||
|
||||
| 层 | 职责 |
|
||||
| --- | --- |
|
||||
| UI 层 | Compose 页面渲染、检测叠加层绘制、交互 |
|
||||
| ViewModel 层 | 页面状态管理、识别结果分发、提醒触发 |
|
||||
| 领域层 | 检测器抽象接口 |
|
||||
| 基础设施层 | CameraX、TFLite、DataStore 等能力实现 |
|
||||
|
||||
### 3.3 核心数据流(识别链路)
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["CameraX 相机帧"] --> B["ImageAnalysis 帧分析"]
|
||||
B --> C["预处理<br/>缩放 320×320 / 归一化 / 旋转校正"]
|
||||
C --> D["TFLite 推理<br/>YOLOv8n"]
|
||||
D --> E["后处理<br/>解码 / NMS / 阈值过滤"]
|
||||
E --> F["叠加层渲染<br/>边框 + 类别 + 置信度"]
|
||||
F --> G["UI 展示"]
|
||||
E --> H["结果提醒<br/>震动 / 提示音"]
|
||||
```
|
||||
|
||||
### 3.4 部署形态
|
||||
|
||||
- **本地优先**:识别全部离线完成,运行期无网络请求;
|
||||
- **云端(V2 可选)**:仅用于模型版本更新下发。
|
||||
|
||||
---
|
||||
|
||||
## 4. 技术选型
|
||||
|
||||
| 类别 | 选型 | 理由 |
|
||||
| --- | --- | --- |
|
||||
| 开发语言 | Kotlin | Android 官方推荐,协程生态成熟 |
|
||||
| UI 框架 | Jetpack Compose(Material 3) | 声明式 UI,开发效率高,叠加层绘制灵活 |
|
||||
| 相机 | CameraX(Preview / ImageAnalysis) | Jetpack 官方库,生命周期安全,机型兼容性最佳 |
|
||||
| 目标检测模型 | YOLOv8n(自定义 4 类训练) | 精度/速度均衡,端侧部署方案成熟 |
|
||||
| 推理框架 | TensorFlow Lite 2.16+(GPU Delegate) | 官方支持 GPU 加速;备选 NCNN / MNN |
|
||||
| 配置存储 | DataStore Preferences | 阈值、开关等设置 |
|
||||
| 异步 | Kotlin Coroutines + Flow | 主线程安全,生命周期感知 |
|
||||
| 构建 | Gradle(Kotlin DSL)+ AGP 8.x | 现代构建配置 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 核心功能技术方案
|
||||
|
||||
### 5.1 实时识别管线
|
||||
|
||||
- CameraX 组合:`Preview`(取景)+ `ImageAnalysis`(识别);
|
||||
- `ImageAnalysis` 使用 `STRATEGY_KEEP_ONLY_LATEST` 背压策略,保证不积压帧;
|
||||
- 输出格式使用 `OUTPUT_IMAGE_FORMAT_RGBA_8888`(CameraX 1.3+),免去 YUV→RGB 手动转换;
|
||||
- **帧节流**:连续模式每 2~3 帧检测一次;标准模式 300ms 一次;省电模式每秒一次(依据设置);
|
||||
- 推理在独立检测线程执行,单例互斥锁防止重叠推理;分析线程不阻塞主线程。
|
||||
|
||||
### 5.2 检测叠加层
|
||||
|
||||
- Compose Canvas 绘制检测框(矩形 + 类别标签 + 置信度),颜色按类别区分;
|
||||
- 检测框内同时标注**大致距离**(如"约 25m"),随检测框实时更新;
|
||||
- 距离标注与检测框同生共灭,展示逻辑一致(≥ 0.5s 出现、消失 2s 后移除);
|
||||
- 坐标映射链路:模型归一化坐标 → 旋转校正(sensor rotation)→ 预览视图坐标(含 FIT_CENTER 裁剪偏移修正);
|
||||
- 目标保持 ≥ 0.5s 才展示,避免单帧误检闪烁。
|
||||
|
||||
### 5.3 距离标注(单目估计)
|
||||
|
||||
- 采用单目针孔模型:`距离 ≈ 焦距px × 物种参考体型 / 检测框像素高度`;
|
||||
- 焦距 px 由相机内参换算(`LENS_INFO_AVAILABLE_FOCAL_LENGTHS` × `SENSOR_INFO_PHYSICAL_SIZE`),个别机型回退用视场角推算;
|
||||
- 物种参考体型内置表(见 6.1),每类一个平均体型值;
|
||||
- 展示为"约 X m";5~50m 范围误差预期 ≤ ±30%;
|
||||
- 纯算术运算,无额外模型与算力开销;V2 可引入地面平面法(设备俯仰角 + 相机高度)提升精度。
|
||||
|
||||
### 5.4 检测提醒
|
||||
|
||||
- 检测到目标时触发震动 / 提示音,用户可开关;
|
||||
- 防重复打扰:10s 内同类目标只提醒一次;
|
||||
- 目标消失后 2s 内不闪断显示,避免频繁提醒。
|
||||
|
||||
### 5.5 低光增强(P2)
|
||||
|
||||
- 简单直方图均衡 / 亮度增益提升低光帧可见度;
|
||||
- V2 可引入夜视增强网络思路。
|
||||
|
||||
---
|
||||
|
||||
## 6. AI 模型方案(核心)
|
||||
|
||||
### 6.1 目标类别定义
|
||||
|
||||
| 类别 ID | 英文标签 | 中文名 | 涵盖范围 | 参考体型(距离估计用) |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 0 | pheasant | 野鸡 | 环颈雉等雉类 | ≈ 0.45m(身高) |
|
||||
|
||||
### 6.2 模型选型对比
|
||||
|
||||
| 模型 | 输入尺寸 | 中端机单帧耗时(GPU) | 精度 | 说明 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| **YOLOv8n(推荐)** | 320~640 | 约 30~80ms | 高 | 支持自定义类别训练,精度/速度平衡 |
|
||||
| EfficientDet-Lite0/2 | 320 | 约 20~50ms | 中 | 仅 COCO 预置类别 |
|
||||
| YOLOv5s | 640 | 约 80~150ms | 高 | 体积与耗电偏大 |
|
||||
| SSD MobileNetV2 | 300 | 约 15~30ms | 低 | 小目标与远距离效果差 |
|
||||
|
||||
**结论**:选用 **YOLOv8n**,使用预训练权重迁移学习自定义训练,导出 TFLite 端侧部署。
|
||||
|
||||
### 6.3 数据集方案
|
||||
|
||||
| 项 | 方案 |
|
||||
| --- | --- |
|
||||
| 数据量 | 1000~2000 张(首版可 500+ 起步,滚动补充) |
|
||||
| 多样性 | 覆盖不同季节、晨昏/正午/逆光、远近距离、姿态、遮挡、背景(草丛/农田/林地/雪地) |
|
||||
| 标注 | Roboflow 或 labelImg,YOLO 格式(class, cx, cy, w, h) |
|
||||
| 数据增强 | Mosaic、MixUp、HSV 扰动、随机翻转、随机缩放裁剪 |
|
||||
| 数据划分 | train 80% / val 10% / test 10% |
|
||||
| 负样本 | 补充无目标场景图,控制误检 |
|
||||
|
||||
### 6.4 训练方案
|
||||
|
||||
- 框架:ultralytics YOLOv8,预训练权重 `yolov8n.pt` 迁移学习;
|
||||
- 超参:imgsz=640(训练)、epochs 100~200(早停)、batch 16~32(视 GPU 而定);
|
||||
- 评估指标:
|
||||
- mAP@0.5 ≥ **0.85**(达标线 0.80);
|
||||
- mAP@0.5:0.95 ≥ **0.55**;
|
||||
- 负样本误检率 ≤ **2%**;
|
||||
- 远距离小目标(高度 ≤ 20px)召回率 ≥ **60%**。
|
||||
|
||||
### 6.5 模型转换与量化
|
||||
|
||||
- 导出命令:`yolo export model=best.pt format=tflite imgsz=320`;
|
||||
- 量化策略:
|
||||
- 优先 **fp16**(配合 GPU Delegate,精度损失小);
|
||||
- 低端机 / CPU 场景使用 **int8** 量化(体积更小、CPU 更快);
|
||||
- 输入尺寸:320×320(连续检测)/ 416×416(远距离模式,V2)。
|
||||
|
||||
### 6.6 推理优化
|
||||
|
||||
- GPU Delegate 优先,初始化失败自动回退 CPU(NNAPI 可选);
|
||||
- 推理线程 4 线程,首次推理前执行预热(dummy run);
|
||||
- 输入输出 ByteBuffer / Bitmap 复用,避免热路径频繁分配。
|
||||
|
||||
### 6.7 精度与速度目标
|
||||
|
||||
| 指标 | 目标 |
|
||||
| --- | --- |
|
||||
| 检测阈值(默认) | confidence ≥ 0.40,IoU-NMS = 0.45 |
|
||||
| 单帧推理延迟 | ≤ 80ms(中端机,320 输入,GPU) |
|
||||
| 模型体积 | fp16 ≤ 8MB,int8 ≤ 4MB |
|
||||
| 识别类别数 | 1 类(野鸡) |
|
||||
| 距离标注 | "约 X m",5~50m 误差 ≤ ±30%,计算开销可忽略 |
|
||||
|
||||
---
|
||||
|
||||
## 7. 性能指标与优化
|
||||
|
||||
| 指标 | 目标值 | 主要优化手段 |
|
||||
| --- | --- | --- |
|
||||
| 启动到相机预览 | ≤ 1.5s | 启动即初始化 CameraProvider,懒加载非核心模块 |
|
||||
| 单帧检测延迟 | ≤ 80ms | GPU Delegate、320 输入、线程复用 |
|
||||
| 预览帧率 | ≥ 25fps | 帧节流、KEEP_ONLY_LATEST、避免主线程工作 |
|
||||
| 内存峰值 | ≤ 200MB | Bitmap/ByteBuffer 复用,无大对象常驻 |
|
||||
| 连续 1 小时耗电 | ≤ 15% | 检测帧节流、省电模式、后台自动释放相机 |
|
||||
| App 体积 | ≤ 40MB | ABI 拆分、模型量化、R8 混淆 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 数据与隐私
|
||||
|
||||
- 无账号、无埋点、无广告 SDK,**不采集任何用户数据**;
|
||||
- 不保存照片、不记录位置,运行期无网络请求;
|
||||
- 模型与标签随 APK 打包,识别数据仅存在于内存中,随进程结束自动释放。
|
||||
|
||||
---
|
||||
|
||||
## 9. 安全与合规
|
||||
|
||||
- 产品定位为**野生动物观察、识别工具**,不提供任何猎捕、诱捕、伤害野生动物的功能或指导;
|
||||
- 遵守《中华人民共和国野生动物保护法》《陆生野生动物保护实施条例》等法律法规;
|
||||
- 野鸡(雉类)属"三有"保护动物,猎捕须依法取得许可,App 不鼓励、不协助非法猎捕;
|
||||
- App 内置观察伦理提示:保持安全距离、不惊扰动物、遵守保护区管理规定(首次启动展示,功能文档 F07)。
|
||||
|
||||
---
|
||||
|
||||
## 10. 开发计划(里程碑)
|
||||
|
||||
| 阶段 | 周期 | 交付内容 |
|
||||
| --- | --- | --- |
|
||||
| M0 准备 | 1 周 | 需求确认、数据集启动采集、Android 工程脚手架、模型基线 |
|
||||
| M1 MVP | 3~4 周 | 相机预览 + 实时检测 + 叠加层 + 检测提醒 + 设置 + 合规提示 |
|
||||
| M2 打磨发布 | 1~2 周 | 性能优化、真机矩阵测试、混淆加固、上架准备 |
|
||||
|
||||
总计约 **5~7 周**(不含数据采集并行时间)。
|
||||
|
||||
---
|
||||
|
||||
## 11. 风险与应对
|
||||
|
||||
| 风险 | 影响 | 应对 |
|
||||
| --- | --- | --- |
|
||||
| 训练数据不足 / 类别相近 | 精度低、误检 | 持续滚动采集数据;增加难例挖掘;灰度发布迭代模型 |
|
||||
| 小目标(远处动物) | 漏检 | 提高输入分辨率;帧节流换取算力;提示用户靠近/变焦;V2 引入 SAHI/tiling 或专用小目标模型 |
|
||||
| 低光 / 夜间场景 | 漏检 | 低光增强预处理;V2 夜视模式 |
|
||||
| 单目距离估计精度有限 | 距离显示不准 | 标注为"约"并明示误差预期;体型表持续校准;V2 地面平面法 + 水下折射修正 |
|
||||
| 生境区域误报偏多 | 提醒频繁、体验差 | 独立低阈值 + 区分提醒方式 + 防重复机制;生境阈值可调;模型迭代降低误报 |
|
||||
| 中低端机型性能不足 | 帧率低、发热 | 动态分辨率、降频检测、GPU 回退策略、省电模式 |
|
||||
| CameraX 个别机型异常 | 黑屏/闪退 | 机型兼容测试矩阵、崩溃监控、失败回退(重试/默认配置) |
|
||||
| 合规风险(被用于非法猎捕) | 法律风险 | 产品定位为观察工具、内置合规提示、不提供猎捕辅助功能 |
|
||||
|
||||
---
|
||||
|
||||
## 12. 验收标准
|
||||
|
||||
1. 野鸡测试集 mAP@0.5 ≥ 0.85,负样本误检率 ≤ 2%;
|
||||
2. 中端机(如骁龙 7 系)单帧检测 ≤ 80ms,预览流畅无卡顿;
|
||||
3. 连续使用 1 小时耗电 ≤ 15%,无异常发热;
|
||||
4. 主流机型(小米 / 华为 / OPPO / vivo / 三星,各 ≥ 2 台)启动成功率 ≥ 99%,崩溃率 ≤ 0.5%;
|
||||
5. 飞行模式下实时识别全流程(预览、检测、提醒、设置)可用;
|
||||
6. 检测框正确显示类别、置信度与距离;5~50m 范围距离误差 ≤ ±30%;
|
||||
7. 通过应用商店合规审核(隐私政策、权限声明完整)。
|
||||
@@ -0,0 +1,193 @@
|
||||
# Observer(野视)· 项目功能文档
|
||||
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 文档版本 | v1.1 |
|
||||
| 编写日期 | 2026-08-17 |
|
||||
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照留存) |
|
||||
| 配套文档 | 《01-技术方案》《03-技术实现文档》 |
|
||||
|
||||
---
|
||||
|
||||
## 1. 产品概述
|
||||
|
||||
### 1.1 产品定位
|
||||
|
||||
一款**纯 Android 原生**的野生动物实时识别 App。用户打开相机,即可从实时画面中识别野鸡,通过彩色检测框(含类别、置信度、距离标注)与震动 / 声音提醒辅助快速发现。**无拍照、无记录、无统计**,专注实时识别这一核心体验,**全程离线可用**。
|
||||
|
||||
### 1.2 目标用户
|
||||
|
||||
- 野生动物观察爱好者(观鸟 / 观兽);
|
||||
- 户外徒步、摄影人群;
|
||||
- 自然教育、科普工作者。
|
||||
|
||||
### 1.3 价值主张
|
||||
|
||||
- **即时发现**:打开相机即识别,检测到目标立即提醒,免去翻图鉴、猜物种;
|
||||
- **零门槛**:不依赖网络,不依赖外设,一部手机即可;
|
||||
- **纯净体验**:无留存功能、无账号、无广告,即开即用。
|
||||
|
||||
---
|
||||
|
||||
## 2. 功能架构(功能树)
|
||||
|
||||
```
|
||||
Observer
|
||||
├── 实时识别
|
||||
│ ├── 相机实时预览(F01)
|
||||
│ ├── 目标实时检测(F02)
|
||||
│ ├── 识别结果展示与距离标注(F03)
|
||||
│ ├── 检测提醒(F04)
|
||||
│ └── 低光增强(F06,P2)
|
||||
├── 设置
|
||||
│ ├── 识别参数(F05)
|
||||
│ └── 提醒设置(F04 子项)
|
||||
└── 合规提示(F07)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 功能需求清单
|
||||
|
||||
优先级定义:**P0**=必须,**P1**=V1.0,**P2**=后续版本。
|
||||
|
||||
| 编号 | 功能 | 功能描述 | 优先级 |
|
||||
| --- | --- | --- | --- |
|
||||
| F01 | 相机实时预览 | 全屏取景,默认后置,支持自动对焦、双指缩放、点击对焦、前后摄切换 | P0 |
|
||||
| F02 | 目标实时检测 | 对野鸡实时检测框选 | P0 |
|
||||
| F03 | 识别结果展示与距离标注 | 检测框 + 类别名 + 置信度百分比 + 大致距离(约 X m);四类目标使用不同颜色边框 | P0 |
|
||||
| F04 | 检测提醒 | 检测到目标时震动 / 提示音;支持开关与 10s 防重复 | P1 |
|
||||
| F05 | 设置 | 置信度阈值(0.2~0.7)、检测频率(连续 / 标准 / 省电)、提醒开关、低光增强开关 | P1 |
|
||||
| F06 | 低光增强 | 低光环境下自动亮度 / 对比度增强,提高识别率 | P2 |
|
||||
| F07 | 合规提示 | 首次启动展示《观察伦理与法律提示》,需用户确认 | P0 |
|
||||
| F08 | 生境区域预警 | 画面中未检测到目标时,识别草丛 / 灌木 / 水面等疑似生境区域,以黄色虚线框展示(含距离标注)并触发预警(独立阈值与提醒方式) | P1 |
|
||||
|
||||
---
|
||||
|
||||
## 4. 核心业务流程
|
||||
|
||||
### 4.1 首次启动与授权
|
||||
|
||||
```
|
||||
启动 App → 合规提示页(F07,用户确认)
|
||||
→ 请求相机权限
|
||||
├─ 授权 → 进入相机主界面
|
||||
└─ 拒绝 → 引导页说明用途,可重新授权
|
||||
```
|
||||
|
||||
### 4.2 实时识别
|
||||
|
||||
```
|
||||
相机主界面(默认后置)
|
||||
→ 画面持续检测(连续 / 标准 / 省电频率)
|
||||
→ 检测到目标:显示彩色检测框 + 类别 + 置信度 + 距离
|
||||
├─ 触发提醒(震动 / 声音,10s 防重复)
|
||||
└─ 目标停留 ≥ 0.5s 保持显示,消失后 2s 内不闪断
|
||||
→ 未检测到目标时:识别疑似生境区域(草丛 / 灌木 / 水面)
|
||||
├─ 黄色虚线框展示,标注"疑似区域"与距离
|
||||
└─ 触发预警(提醒方式与动物检测区分)
|
||||
→ 用户可点击检测框查看识别详情(物种、置信度、时间)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 界面设计
|
||||
|
||||
### 5.1 界面总览
|
||||
|
||||
仅两个界面:**相机主界面**(默认全屏展示)+ **设置页**(右上角齿轮入口)。
|
||||
|
||||
### 5.2 相机主界面(核心界面)
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
│ [低光] [频率] [前/后摄] │ ← 顶部工具
|
||||
│ 相机预览画面(全屏) │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ ┌───────┐ │ │
|
||||
│ │ │ 野鸡 86% │ │ │ ← 检测框(按类别着色)
|
||||
│ │ │ 约 25m │ │ │
|
||||
│ │ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ │ │ ← 疑似生境区域(黄色虚线,预警)
|
||||
│ │ 疑似区域 · 约 20m │ │
|
||||
│ │ └───────┘ │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ [提醒开关] [设置] │ ← 底部操作
|
||||
└─────────────────────────────────┘
|
||||
```
|
||||
|
||||
交互说明:
|
||||
|
||||
- 顶部:检测频率切换(连续 / 标准 / 省电)、低光增强开关(P2)、前后摄像头切换;
|
||||
- 中部:全屏预览,检测框 + 类别 + 置信度实时叠加;
|
||||
- 底部:提醒开关(震动 / 声音)、设置入口;
|
||||
- 支持双指缩放、点击对焦;
|
||||
- 检测到目标时边框高亮 + 震动 / 声音提醒(10s 防重复);
|
||||
- 检测框内显示类别、置信度与大致距离(约 X m),随目标实时更新;
|
||||
- 未检测到目标时,疑似生境区域以黄色虚线框展示(含距离标注)并预警(提醒方式与动物检测区分)。
|
||||
|
||||
### 5.3 识别结果卡片(点击检测框弹出)
|
||||
|
||||
| 元素 | 说明 |
|
||||
| --- | --- |
|
||||
| 物种名称 | 中文名 + 英文标签(如:野鸡 pheasant) |
|
||||
| 置信度 | 百分比进度条 |
|
||||
| 时间 | 检测时刻 |
|
||||
| 距离 | 约 25m(单目估算,误差 ±30%) |
|
||||
|
||||
仅展示信息,无保存操作。
|
||||
|
||||
疑似生境区域(黄色虚线框)点击后展示:区域类别、置信度、距离(约 X m,按参考植被高度估算,误差较大)。
|
||||
|
||||
### 5.4 设置页
|
||||
|
||||
| 分组 | 设置项 | 默认值 |
|
||||
| --- | --- | --- |
|
||||
| 识别 | 置信度阈值 | 0.40 |
|
||||
| 识别 | 检测频率 | 连续 |
|
||||
| 识别 | 低光增强 | 关(P2) |
|
||||
| 识别 | 距离标注 | 开 |
|
||||
| 识别 | 生境区域阈值 | 0.35 |
|
||||
| 提醒 | 震动提醒 / 提示音 / 防重复时长 | 开 / 开 / 10s |
|
||||
| 提醒 | 生境区域预警 | 开 |
|
||||
| 关于 | 版本信息 / 隐私说明 | — |
|
||||
|
||||
---
|
||||
|
||||
## 6. 权限设计
|
||||
|
||||
| 权限 | 用途 | 时机 | 备注 |
|
||||
| --- | --- | --- | --- |
|
||||
| CAMERA | 实时预览与识别 | 首次启动 | 必须 |
|
||||
|
||||
原则:**最小权限、按需申请**。仅申请相机权限;拒绝后提供说明引导页(可跳转系统设置重新授权)。不申请定位、存储等任何其他权限。
|
||||
|
||||
---
|
||||
|
||||
## 7. 非功能需求
|
||||
|
||||
| 类别 | 要求 |
|
||||
| --- | --- |
|
||||
| 兼容性 | Android 7.0+(minSdk 24),竖屏为主,支持暗色模式 |
|
||||
| 性能 | 见《技术方案》第 7 章(延迟 ≤ 80ms、启动 ≤ 1.5s、内存 ≤ 200MB) |
|
||||
| 离线 | 飞行模式下实时识别全流程完整可用 |
|
||||
| 稳定性 | 崩溃率 ≤ 0.5%,启动成功率 ≥ 99%,异常自动降级(GPU 回退 CPU) |
|
||||
| 耗电 | 连续使用 1 小时 ≤ 15%;省电模式 ≤ 8% |
|
||||
| 隐私 | 无广告 SDK、无埋点、不保存任何数据,识别数据仅存内存 |
|
||||
| 无障碍 | 关键操作支持 TalkBack 描述;检测结果支持语音播报(V2) |
|
||||
|
||||
---
|
||||
|
||||
## 8. 版本规划
|
||||
|
||||
| 版本 | 范围 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| MVP / V1.0 | F01~F05、F07、F08 | 相机 + 实时识别 + 生境预警 + 提醒 + 设置,即完整核心体验 |
|
||||
| V2.0 | +F06 及增强 | 低光增强、更多物种、夜视模式、模型远程更新、语音播报、距离精度增强(地面平面法) |
|
||||
|
||||
---
|
||||
|
||||
## 9. 合规与安全说明
|
||||
|
||||
- App 定位为**野生动物观察、识别工具**,不提供猎捕、诱捕、伤害野生动物的功能或指导;
|
||||
- 首次启动展示合规提示:遵守《中华人民共和国野生动物保护法》,野鸡属"三有"保护动物,猎捕须依法许可;观察时保持距离、不惊扰动物、遵守保护区规定;
|
||||
- 上架需提供隐私政策,声明仅使用相机权限、不采集与存储任何用户数据。
|
||||
@@ -0,0 +1,512 @@
|
||||
# Observer(野视)· 技术实现文档
|
||||
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 文档版本 | v1.1 |
|
||||
| 编写日期 | 2026-08-17 |
|
||||
| 状态 | 初稿(v1.1:确认功能范围为"仅实时识别",移除拍照、记录、相册识别、定位相关实现) |
|
||||
| 配套文档 | 《01-技术方案》《02-项目功能文档》 |
|
||||
|
||||
---
|
||||
|
||||
## 1. 项目结构
|
||||
|
||||
```
|
||||
app/
|
||||
├── src/main/
|
||||
│ ├── java/com/example/observer/
|
||||
│ │ ├── MainActivity.kt // 单 Activity 入口
|
||||
│ │ ├── camera/
|
||||
│ │ │ ├── CameraController.kt // CameraX 生命周期绑定与用例组合
|
||||
│ │ │ └── FrameAnalyzer.kt // ImageAnalysis 帧分析器(节流 + 调度)
|
||||
│ │ ├── detection/
|
||||
│ │ │ ├── Detector.kt // 检测器抽象接口
|
||||
│ │ │ ├── TFLiteDetector.kt // TFLite 实现(YOLOv8n)
|
||||
│ │ │ ├── DetectionResult.kt // 检测结果数据类
|
||||
│ │ │ ├── Nms.kt // NMS 后处理
|
||||
│ │ │ └── CoordinateMapper.kt // 模型坐标 → 视图坐标映射
|
||||
│ │ ├── distance/
|
||||
│ │ │ └── DistanceEstimator.kt // 单目距离估计(针孔模型)
|
||||
│ │ ├── overlay/
|
||||
│ │ │ └── DetectionOverlay.kt // Compose 叠加层(Canvas 绘制检测框)
|
||||
│ │ ├── reminder/
|
||||
│ │ │ └── Reminder.kt // 震动 / 提示音提醒(含防重复)
|
||||
│ │ ├── ui/
|
||||
│ │ │ ├── camera/ CameraScreen.kt / CameraViewModel.kt
|
||||
│ │ │ └── settings/ SettingsScreen.kt / SettingsViewModel.kt
|
||||
│ │ └── util/
|
||||
│ │ └── BitmapUtils.kt // 缩放 / 旋转 / 复用
|
||||
│ ├── assets/
|
||||
│ │ ├── model.tflite // YOLOv8n 四类模型
|
||||
│ │ └── labels.txt // 类别标签(按 ID 顺序)
|
||||
│ └── res/
|
||||
└── build.gradle.kts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 技术栈与版本
|
||||
|
||||
| 组件 | 版本(以最新稳定为准) | 用途 |
|
||||
| --- | --- | --- |
|
||||
| Kotlin | 2.x | 开发语言 |
|
||||
| AGP | 8.x | Android Gradle 插件 |
|
||||
| Jetpack Compose | BOM 2024.x+(Material 3) | UI |
|
||||
| CameraX | 1.4.x | Preview / ImageAnalysis |
|
||||
| TensorFlow Lite | 2.16.x | 端侧推理 |
|
||||
| tensorflow-lite-gpu | 2.16.x | GPU 加速 |
|
||||
| DataStore | 1.1.x | 设置存储 |
|
||||
|
||||
依赖(build.gradle.kts 关键片段):
|
||||
|
||||
```kotlin
|
||||
dependencies {
|
||||
implementation("androidx.camera:camera-core:1.4.1")
|
||||
implementation("androidx.camera:camera-camera2:1.4.1")
|
||||
implementation("androidx.camera:camera-lifecycle:1.4.1")
|
||||
implementation("androidx.camera:camera-view:1.4.1")
|
||||
|
||||
implementation("org.tensorflow:tensorflow-lite:2.16.1")
|
||||
implementation("org.tensorflow:tensorflow-lite-gpu:2.16.1")
|
||||
|
||||
implementation("androidx.datastore:datastore-preferences:1.1.1")
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 模块设计
|
||||
|
||||
### 3.1 camera 模块 — CameraController.kt
|
||||
|
||||
职责:创建并绑定 CameraX 用例(Preview + ImageAnalysis),统一生命周期。
|
||||
|
||||
```kotlin
|
||||
class CameraController(
|
||||
private val lifecycleOwner: LifecycleOwner,
|
||||
private val analysisExecutor: Executor,
|
||||
) {
|
||||
private lateinit var cameraProvider: ProcessCameraProvider
|
||||
private lateinit var imageAnalysis: ImageAnalysis
|
||||
|
||||
fun start(previewView: PreviewView, analyzer: ImageAnalysis.Analyzer) {
|
||||
val future = ProcessCameraProvider.getInstance(previewView.context)
|
||||
future.addListener({
|
||||
cameraProvider = future.get()
|
||||
|
||||
val preview = Preview.Builder().build().also {
|
||||
it.surfaceProvider = previewView.surfaceProvider
|
||||
}
|
||||
|
||||
imageAnalysis = ImageAnalysis.Builder()
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
|
||||
.build()
|
||||
imageAnalysis.setAnalyzer(analysisExecutor, analyzer)
|
||||
|
||||
cameraProvider.bindToLifecycle(
|
||||
lifecycleOwner,
|
||||
CameraSelector.DEFAULT_BACK_CAMERA,
|
||||
preview, imageAnalysis,
|
||||
)
|
||||
}, ContextCompat.getMainExecutor(previewView.context))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
要点:
|
||||
|
||||
- `OUTPUT_IMAGE_FORMAT_RGBA_8888`(CameraX 1.3+)直接得到 RGBA 图像,免 YUV 转换;
|
||||
- `STRATEGY_KEEP_ONLY_LATEST`:分析器忙时丢弃旧帧,不积压;
|
||||
- 相机权限检查与请求在进入本模块前完成;
|
||||
- 前后摄切换:重新以对应 `CameraSelector` 执行 `bindToLifecycle`(解绑旧用例)。
|
||||
|
||||
### 3.2 camera 模块 — FrameAnalyzer.kt
|
||||
|
||||
职责:帧节流、旋转获取、调度检测、结果回调。
|
||||
|
||||
```kotlin
|
||||
class FrameAnalyzer(
|
||||
private val detector: Detector,
|
||||
private val onResult: (List<DetectionResult>, Int) -> Unit, // 结果 + 旋转角
|
||||
) : ImageAnalysis.Analyzer {
|
||||
|
||||
private var lastDetectMs = 0L
|
||||
|
||||
override fun analyze(imageProxy: ImageProxy) {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
val interval = frameIntervalMs() // 依据设置:连续/标准/省电
|
||||
if (now - lastDetectMs < interval) {
|
||||
imageProxy.close(); return
|
||||
}
|
||||
lastDetectMs = now
|
||||
|
||||
val bitmap = imageProxy.toBitmap() // RGBA_8888 直接转 Bitmap
|
||||
val rotation = imageProxy.imageInfo.rotationDegrees
|
||||
try {
|
||||
val results = detector.detect(bitmap)
|
||||
onResult(results, rotation)
|
||||
} finally {
|
||||
imageProxy.close() // 必须关闭,否则阻塞流
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.3 detection 模块 — Detector 接口
|
||||
|
||||
```kotlin
|
||||
interface Detector {
|
||||
/** 输入 RGBA 位图,输出归一化坐标检测结果(0..1) */
|
||||
fun detect(bitmap: Bitmap): List<DetectionResult>
|
||||
}
|
||||
```
|
||||
|
||||
### 3.4 detection 模块 — TFLiteDetector.kt
|
||||
|
||||
```kotlin
|
||||
class TFLiteDetector(
|
||||
context: Context,
|
||||
private val inputSize: Int = 320,
|
||||
private val confThreshold: Float = 0.40f, // 动物类阈值
|
||||
private val habitatThreshold: Float = 0.35f, // 生境区域阈值
|
||||
private val iouThreshold: Float = 0.45f,
|
||||
gpuEnabled: Boolean = true,
|
||||
) : Detector {
|
||||
|
||||
private val labels: List<String> =
|
||||
context.assets.open("labels.txt").bufferedReader().readLines()
|
||||
|
||||
private val interpreter: Interpreter = Interpreter(
|
||||
loadModelFile(context, "model.tflite"),
|
||||
Interpreter.Options().apply {
|
||||
setNumThreads(4)
|
||||
if (gpuEnabled) {
|
||||
try { addDelegate(GpuDelegate()) } catch (_: Exception) { /* 回退 CPU */ }
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
private val inputBuffer: ByteBuffer = ByteBuffer.allocateDirect(
|
||||
1 * inputSize * inputSize * 3 * 4 // float32
|
||||
).order(ByteOrder.nativeOrder())
|
||||
|
||||
private val outputBuffer: ByteBuffer = ByteBuffer.allocateDirect(
|
||||
OUTPUT_ELEMENTS * 4
|
||||
).order(ByteOrder.nativeOrder())
|
||||
|
||||
override fun detect(bitmap: Bitmap): List<DetectionResult> {
|
||||
preprocess(bitmap, inputBuffer) // 缩放 + RGB 归一化 → 输入缓冲
|
||||
interpreter.run(inputBuffer, outputBuffer) // 单张推理
|
||||
return postprocess(outputBuffer, bitmap.width, bitmap.height)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
输入 / 输出规格(YOLOv8n 四类,320 输入):
|
||||
|
||||
- 输入:`[1, 320, 320, 3]`,RGB,float32 归一化 0~1;
|
||||
- 输出:`[1, 9, 8400]`(8400 = 各尺度 anchor 数,9 = 4 个框坐标 cx/cy/w/h + 5 类得分),展平为 `9 * 8400` 个 float。
|
||||
|
||||
### 3.5 detection 模块 — 后处理(解码 + NMS)
|
||||
|
||||
```kotlin
|
||||
private fun postprocess(raw: FloatArray, imgW: Int, imgH: Int): List<DetectionResult> {
|
||||
val numAnchor = raw.size / 9
|
||||
val boxes = mutableListOf<DetectionResult>()
|
||||
|
||||
for (a in 0 until numAnchor) {
|
||||
val cx = raw[a]; val cy = raw[9 + a]
|
||||
val w = raw[18 + a]; val h = raw[27 + a]
|
||||
var bestCls = 0; var bestScore = 0f
|
||||
for (c in 0 until 5) {
|
||||
val s = raw[36 + c * numAnchor + a] // 类别得分按 anchor 平铺
|
||||
if (s > bestScore) { bestScore = s; bestCls = c }
|
||||
}
|
||||
if (bestScore < confThreshold) continue
|
||||
boxes += DetectionResult(
|
||||
label = labels[bestCls],
|
||||
score = bestScore,
|
||||
// 归一化坐标(裁剪到 [0,1])
|
||||
left = (cx - w / 2).coerceIn(0f, 1f),
|
||||
top = (cy - h / 2).coerceIn(0f, 1f),
|
||||
right = (cx + w / 2).coerceIn(0f, 1f),
|
||||
bottom = (cy + h / 2).coerceIn(0f, 1f),
|
||||
)
|
||||
}
|
||||
return nms(boxes, iouThreshold)
|
||||
}
|
||||
|
||||
fun nms(boxes: List<DetectionResult>, iouThreshold: Float): List<DetectionResult> {
|
||||
val sorted = boxes.sortedByDescending { it.score }
|
||||
val kept = mutableListOf<DetectionResult>()
|
||||
for (b in sorted) {
|
||||
if (kept.none { iou(b, it) > iouThreshold }) kept += b
|
||||
}
|
||||
return kept
|
||||
}
|
||||
```
|
||||
|
||||
注意:以上取数索引为示意,**必须与模型导出时的输出布局(yolov8 tflite 为 9×8400,按列平铺)核对一致**,建议训练导出后用 Python 脚本先对单图做一致性校验再接入 App。
|
||||
|
||||
### 3.6 distance 模块 — DistanceEstimator.kt
|
||||
|
||||
```kotlin
|
||||
class DistanceEstimator(context: Context) {
|
||||
|
||||
private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
|
||||
private var focalPxCache = -1f
|
||||
|
||||
// 物种参考体型(米),用于针孔模型估算
|
||||
private val speciesSizeM = mapOf(
|
||||
"pheasant" to 0.45f, // 身高
|
||||
)
|
||||
|
||||
/** 距离 = 焦距px × 参考体型 / 框高px(在分析图像分辨率下计算) */
|
||||
fun estimate(label: String, boxHeightNorm: Float, imageHeightPx: Int): Float? {
|
||||
val realH = speciesSizeM[label] ?: return null
|
||||
val boxH = boxHeightNorm * imageHeightPx
|
||||
if (boxH < 8f) return null // 过小目标不估算
|
||||
val focalPx = focalPx(imageHeightPx)
|
||||
if (focalPx <= 0) return null
|
||||
return round(focalPx * realH / boxH)
|
||||
}
|
||||
|
||||
/** focal_px = focal_mm × (imageHeightPx / sensorHeightMm);视场角作回退 */
|
||||
private fun focalPx(imageHeightPx: Int): Float {
|
||||
if (focalPxCache > 0) return focalPxCache
|
||||
val c = cameraManager.getCameraCharacteristics(cameraManager.cameraIdList.first())
|
||||
val focalMm = c.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)?.firstOrNull()
|
||||
val sensor = c.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)
|
||||
val fov = c.get(CameraCharacteristics.LENS_INFO_HORIZONTAL_VIEW_ANGLE)
|
||||
focalPxCache = when {
|
||||
focalMm != null && sensor != null -> focalMm * imageHeightPx / sensor.height
|
||||
fov != null -> (imageHeightPx / 2f) / tan(fov / 2f)
|
||||
else -> -1f
|
||||
}
|
||||
return focalPxCache
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- 前后摄切换时需按当前相机重新获取焦距(缓存按相机 ID 区分);
|
||||
- 距离估算仅供参考,实际距离可能因环境因素有所偏差。
|
||||
|
||||
### 3.7 detection 模块 — CoordinateMapper.kt
|
||||
|
||||
模型输出为归一化坐标(相对分析图像,竖屏方向),需转换到预览视图坐标:
|
||||
|
||||
```kotlin
|
||||
class CoordinateMapper(private val view: PreviewView) {
|
||||
|
||||
/** 归一化坐标 → 预览视图像素坐标(考虑传感器旋转与 FIT_CENTER 裁剪) */
|
||||
fun mapToView(norm: DetectionResult, rotation: Int): RectF {
|
||||
// 1) 旋转校正:把"图像方向"归一化坐标转到"竖屏视图方向"
|
||||
val (x0, y0, x1, y1) = rotate(norm, rotation)
|
||||
// 2) 处理 FIT_CENTER 的裁剪偏移与缩放
|
||||
val viewW = view.width; val viewH = view.height
|
||||
val scale = min(viewW / bitmapW, viewH / bitmapH) // bitmap 尺寸
|
||||
val offsetX = (viewW - bitmapW * scale) / 2f
|
||||
val offsetY = (viewH - bitmapH * scale) / 2f
|
||||
return RectF(
|
||||
x0 * bitmapW * scale + offsetX,
|
||||
y0 * bitmapH * scale + offsetY,
|
||||
x1 * bitmapW * scale + offsetX,
|
||||
y1 * bitmapH * scale + offsetY,
|
||||
)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.8 overlay 模块 — DetectionOverlay.kt
|
||||
|
||||
Compose Canvas 叠加层:
|
||||
|
||||
```kotlin
|
||||
@Composable
|
||||
fun DetectionOverlay(results: List<DetectionResult>, rotation: Int, modifier: Modifier) {
|
||||
val mapper = remember { CoordinateMapper(view) }
|
||||
Canvas(modifier = modifier) {
|
||||
results.forEach { r ->
|
||||
val rect = mapper.mapToView(r, rotation)
|
||||
drawRect(
|
||||
color = colorOf(r.label),
|
||||
topLeft = Offset(rect.left, rect.top),
|
||||
size = Size(rect.width(), rect.height()),
|
||||
style = Stroke(6.dp.toPx()),
|
||||
)
|
||||
// 距离由 ViewModel 计算后写入 DetectionResult.distance(Float?)
|
||||
val dist = r.distance?.let { " · 约${it}m" } ?: ""
|
||||
drawText("${r.label} ${(r.score * 100).toInt()}%$dist")
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
防闪烁:ViewModel 中对结果做"目标停留 ≥ 0.5s 才显示、消失 2s 后移除"的平滑处理。
|
||||
|
||||
### 3.9 reminder 模块 — Reminder.kt
|
||||
|
||||
```kotlin
|
||||
class Reminder(private val context: Context) {
|
||||
private val vibrator = context.getSystemService(Vibrator::class.java)
|
||||
private var lastAlertAt = 0L
|
||||
private var lastAlertLabel: String? = null
|
||||
|
||||
/** 同类目标 10s 内只提醒一次 */
|
||||
fun onDetected(label: String) {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
if (label == lastAlertLabel && now - lastAlertAt < 10_000) return
|
||||
lastAlertAt = now
|
||||
lastAlertLabel = label
|
||||
vibrator?.vibrate(VibrationEffect.createOneShot(200, VibrationEffect.DEFAULT_AMPLITUDE))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.10 设置存储 — DataStore
|
||||
|
||||
使用 DataStore Preferences 保存:`conf_threshold`(默认 0.40)、`detect_mode`(连续/标准/省电)、`vibrate_enabled`、`sound_enabled`、`low_light_enhance`(P2)、`show_distance`(默认开)。`SettingsViewModel` 以 Flow 暴露,`FrameAnalyzer` 与 `TFLiteDetector` 读取最新值。
|
||||
|
||||
---
|
||||
|
||||
## 4. 关键流程实现
|
||||
|
||||
### 4.1 启动流程
|
||||
|
||||
```
|
||||
MainActivity.onCreate
|
||||
→ 检查/请求 CAMERA 权限
|
||||
→ 初始化 TFLiteDetector(后台线程,含预热推理)
|
||||
→ CameraController.start(previewView, analyzer)
|
||||
→ ViewModel 订阅检测结果 → 叠加层渲染 + 提醒触发
|
||||
```
|
||||
|
||||
预热:加载模型后执行一次空推理(全零输入),避免首帧卡顿。
|
||||
|
||||
### 4.2 帧节流与并发控制
|
||||
|
||||
- 检测间隔:连续 80~100ms / 标准 300ms / 省电 1000ms(由设置决定);
|
||||
- 推理在单线程 Executor(HandlerThread)串行执行,天然互斥;
|
||||
- 分析线程只做"取帧 → 判断节流 → 提交任务",不做推理,保证预览流畅。
|
||||
|
||||
### 4.3 提醒、距离标注与防闪烁
|
||||
|
||||
- 检测到目标(置信度 ≥ 阈值,生境区域用独立阈值)→ `Reminder.onDetected(label)`,10s 防重复,动物与生境预警提醒方式区分;
|
||||
- 叠加层展示逻辑:目标连续出现 ≥ 0.5s 才绘制;消失后 2s 内不清除,避免闪烁;
|
||||
- 低置信度结果(阈值以下)不展示、不提醒;
|
||||
- 距离标注:结果到达后调用 `DistanceEstimator` 估算"约 X m",随检测框渲染(FrameAnalyzer 回调携带分析图像高度);生境区域同样估算(按参考植被高度,误差较大);框高过小(< 8px)或无法获取焦距显示"--"。
|
||||
|
||||
---
|
||||
|
||||
## 5. 模型集成细节
|
||||
|
||||
| 项 | 说明 |
|
||||
| --- | --- |
|
||||
| 模型文件 | `app/src/main/assets/model.tflite`,随 APK 打包 |
|
||||
| 标签文件 | `assets/labels.txt`,每行一个类别,顺序与训练一致(pheasant) |
|
||||
| 量化 | 优先 fp16(GPU);int8 用于 CPU/低端机回退 |
|
||||
| 加载 | `Interpreter(loadAssetFile(...))`,进程内单例 |
|
||||
| 预热 | 初始化后执行一次 dummy run |
|
||||
| 校验 | 发布前用 Python 脚本(tflite-runtime)对测试集抽样验证模型输出布局与 App 解析一致 |
|
||||
|
||||
---
|
||||
|
||||
## 6. 性能优化实践
|
||||
|
||||
| 手段 | 说明 |
|
||||
| --- | --- |
|
||||
| GPU Delegate | 优先启用;`addDelegate` 抛异常或首帧异常时回退 CPU,并上报降级日志 |
|
||||
| 输入缓冲复用 | ByteBuffer 复用,避免每次检测重新分配 |
|
||||
| 帧节流 | 按模式降频,KEEP_ONLY_LATEST 不积压 |
|
||||
| 位图复用 | ImageProxy → Bitmap 走共享缓冲;叠加层仅绘制检测框,不复制整帧 |
|
||||
| 线程模型 | 检测线程单线程串行;UI 线程零推理 |
|
||||
| 低分辨率检测 | 320 输入检测;远距离模式可切 416(V2) |
|
||||
| 距离标注 | 纯算术运算开销可忽略;焦距 px 按相机缓存复用,避免重复查询 CameraCharacteristics |
|
||||
| 生命周期 | 后台自动 `unbind` 相机释放资源,避免耗电 |
|
||||
| 省电模式 | 降到 1 帧/秒 + 低分辨率 + 关闭提醒以外的动画 |
|
||||
|
||||
---
|
||||
|
||||
## 7. 异常与降级策略
|
||||
|
||||
| 异常场景 | 处理 |
|
||||
| --- | --- |
|
||||
| GPU 不可用 / 模型加载失败 | 回退 CPU 线程数调整;失败则提示"识别不可用"但不影响相机预览 |
|
||||
| 相机初始化失败(个别机型) | 重试一次 → 失败提示并引导检查权限/重启 |
|
||||
| 权限被拒 | 引导页说明用途,提供跳转系统设置 |
|
||||
| 低内存 | 主动释放缓存位图,降低检测分辨率 |
|
||||
| 推理超时(> 500ms) | 丢弃该帧结果,恢复下一帧,保证预览流畅 |
|
||||
| 无法读取焦距 / 视场角(个别机型) | 距离显示"--",识别不受影响 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 测试方案
|
||||
|
||||
### 8.1 单元测试(JUnit)
|
||||
|
||||
- `NmsTest`:NMS 正确性(重叠/多类别/边界框);
|
||||
- `CoordinateMapperTest`:四向旋转映射、FIT_CENTER 裁剪偏移;
|
||||
- `ThresholdTest`:置信度阈值过滤逻辑;
|
||||
- `ReminderTest`:10s 防重复提醒逻辑、动物与生境提醒方式区分;
|
||||
- `DistanceEstimatorTest`:焦距换算、距离公式、过小目标不估算、生境区域按植被高度估算。
|
||||
|
||||
### 8.2 模型评估(Python 脚本,独立于 App)
|
||||
|
||||
- 对测试集计算 mAP@0.5、mAP@0.5:0.95、各类别 AP、负样本误检率;
|
||||
- 输出混淆矩阵,分析检测精度。
|
||||
|
||||
### 8.3 仪器测试(androidTest)
|
||||
|
||||
- 模拟帧注入:将测试图片经 `ImageProxy` 注入分析器,断言结果(不依赖真机取景);
|
||||
- 权限拒绝 / 恢复场景;
|
||||
- 冷启动到预览的时间基准测试。
|
||||
|
||||
### 8.4 真机测试矩阵
|
||||
|
||||
| 机型档位 | 代表机型 | 验证项 |
|
||||
| --- | --- | --- |
|
||||
| 旗舰 | 骁龙 8 系 / 麒麟 9 系 | 全功能、GPU 路径、长时间发热 |
|
||||
| 中端 | 骁龙 7 系 / 天玑 8 系 | 延迟 ≤ 80ms、耗电 ≤ 15%/h |
|
||||
| 低端 | 骁龙 4 系 / 天玑 6 系 | CPU 回退路径、省电模式可用性 |
|
||||
|
||||
---
|
||||
|
||||
## 9. 构建与发布
|
||||
|
||||
```kotlin
|
||||
android {
|
||||
compileSdk = 35
|
||||
defaultConfig {
|
||||
applicationId = "com.example.observer"
|
||||
minSdk = 24
|
||||
targetSdk = 35
|
||||
ndk { abiFilters += listOf("arm64-v8a") } // 主发 arm64;如需兼容 32 位另发包
|
||||
}
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = true
|
||||
isShrinkResources = true
|
||||
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- R8 规则:保留 TFLite 相关类(`-keep class org.tensorflow.** { *; }`),assets 模型不混淆;
|
||||
- 签名:正式签名 + Gradle 管理(或环境变量注入),不上传 keystore 到仓库;
|
||||
- 上架检查:隐私政策(仅相机权限、不采集与存储数据声明)、权限用途说明。
|
||||
|
||||
---
|
||||
|
||||
## 10. 后续演进(V2 方向)
|
||||
|
||||
- 更多物种类别(哺乳类、鸟类细分),支持模型远程更新;
|
||||
- 小目标优化:SAHI 切图推理 / 专用小目标模型 / 数字变焦辅助;
|
||||
- 夜视 / 红外增强模式;
|
||||
- 检测结果语音播报,提升无障碍体验;
|
||||
- 距离精度增强:基于设备俯仰角与相机高度的地面平面法、镜头畸变校正、水下折射修正;
|
||||
- 生境类别细分与精度提升(湿地、林缘等),生境预警策略优化;
|
||||
- 若未来需要留存能力(拍照、记录),可基于现有检测链路平滑扩展。
|
||||
@@ -0,0 +1,4 @@
|
||||
org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
|
||||
android.useAndroidX=true
|
||||
kotlin.code.style=official
|
||||
android.nonTransitiveRClass=true
|
||||
@@ -0,0 +1,39 @@
|
||||
[versions]
|
||||
agp = "8.5.2"
|
||||
kotlin = "2.0.20"
|
||||
coreKtx = "1.13.1"
|
||||
lifecycle = "2.8.6"
|
||||
activityCompose = "1.9.2"
|
||||
composeBom = "2024.09.03"
|
||||
camerax = "1.4.1"
|
||||
tflite = "2.16.1"
|
||||
datastore = "1.1.1"
|
||||
coroutines = "1.9.0"
|
||||
junit = "4.13.2"
|
||||
|
||||
[libraries]
|
||||
androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
|
||||
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
||||
androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycle" }
|
||||
androidx-lifecycle-runtime-compose = { group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" }
|
||||
androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" }
|
||||
androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" }
|
||||
androidx-compose-ui = { group = "androidx.compose.ui", name = "ui" }
|
||||
androidx-compose-ui-graphics = { group = "androidx.compose.ui", name = "ui-graphics" }
|
||||
androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling" }
|
||||
androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview" }
|
||||
androidx-compose-material3 = { group = "androidx.compose.material3", name = "material3" }
|
||||
androidx-camera-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" }
|
||||
androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" }
|
||||
androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" }
|
||||
androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" }
|
||||
org-tensorflow-lite = { group = "org.tensorflow", name = "tensorflow-lite", version.ref = "tflite" }
|
||||
org-tensorflow-lite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tflite" }
|
||||
androidx-datastore-preferences = { group = "androidx.datastore", name = "datastore-preferences", version.ref = "datastore" }
|
||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" }
|
||||
junit = { group = "junit", name = "junit", version.ref = "junit" }
|
||||
|
||||
[plugins]
|
||||
android-application = { id = "com.android.application", version.ref = "agp" }
|
||||
kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
|
||||
kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }
|
||||
Vendored
BIN
Binary file not shown.
+9
@@ -0,0 +1,9 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
|
||||
networkTimeout=10000
|
||||
retries=0
|
||||
retryBackOffMs=500
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# gradlew start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh gradlew
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
@rem SPDX-License-Identifier: Apache-2.0
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem gradlew startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables, and ensure extensions are enabled
|
||||
setlocal EnableExtensions
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
|
||||
|
||||
@rem Execute gradlew
|
||||
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
|
||||
@rem which allows us to clear the local environment before executing the java command
|
||||
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
|
||||
|
||||
:exitWithErrorLevel
|
||||
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
|
||||
"%COMSPEC%" /c exit %ERRORLEVEL%
|
||||
@@ -0,0 +1,24 @@
|
||||
pluginManagement {
|
||||
repositories {
|
||||
google {
|
||||
content {
|
||||
includeGroupByRegex("com\\.android.*")
|
||||
includeGroupByRegex("com\\.google.*")
|
||||
includeGroupByRegex("androidx.*")
|
||||
}
|
||||
}
|
||||
mavenCentral()
|
||||
gradlePluginPortal()
|
||||
}
|
||||
}
|
||||
|
||||
dependencyResolutionManagement {
|
||||
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
|
||||
repositories {
|
||||
google()
|
||||
mavenCentral()
|
||||
}
|
||||
}
|
||||
|
||||
rootProject.name = "Observer"
|
||||
include(":app")
|
||||
@@ -0,0 +1,202 @@
|
||||
"""用 LocalAI qwen3.5-9b 多模态模型自动标注图片。
|
||||
|
||||
用法:python auto_label.py
|
||||
输入:datasets/images/<class>/*.jpg
|
||||
输出:datasets/dataset/{train,val}/{images,labels} (YOLO 格式) + data.yaml
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import base64
|
||||
import shutil
|
||||
import sys
|
||||
import urllib.request
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
# LocalAI 服务器配置
|
||||
LOCAL_AI_URL = "http://192.168.3.210:18080/v1/chat/completions"
|
||||
MODEL_NAME = "qwen3.5-9b"
|
||||
|
||||
# 标注配置
|
||||
KEEP_PER_CLASS = 100
|
||||
MIN_CONF = 0.5 # 自动标注最低置信度
|
||||
MIN_BOX = 0.03 # 框面积占比下限
|
||||
|
||||
BASE = os.path.dirname(__file__)
|
||||
IMG_DIR = os.path.join(BASE, "datasets", "images")
|
||||
OUT_DIR = os.path.join(BASE, "datasets", "dataset")
|
||||
PREVIEW_DIR = os.path.join(BASE, "datasets", "preview")
|
||||
|
||||
CLASSES = ["pheasant"]
|
||||
PROMPTS = {
|
||||
"pheasant": "pheasant",
|
||||
}
|
||||
|
||||
# 标注提示词模板
|
||||
ANNOTATION_PROMPT = """你是一个目标检测助手。请检测图片中【{target}】动物。
|
||||
|
||||
输出格式(严格只输出这一行,不要其他文字):
|
||||
pheasant 0.92 0.1 0.2 0.3 0.4
|
||||
|
||||
字段说明:
|
||||
- 第一个数字:置信度(0-1)
|
||||
- 后四个数字:边界框坐标 x1 y1 x2 y2(归一化 0-1)
|
||||
|
||||
如果没找到目标,只输出:NOT_FOUND
|
||||
"""
|
||||
|
||||
|
||||
def encode_image_to_base64(path: str) -> str:
|
||||
"""将图片路径转换为 base64 数据"""
|
||||
with open(path, "rb") as f:
|
||||
return base64.b64encode(f.read()).decode("utf-8")
|
||||
|
||||
|
||||
def call_localai(image_path: str, prompt: str) -> dict:
|
||||
"""调用 LocalAI qwen3.5-9b 进行标注"""
|
||||
image_b64 = encode_image_to_base64(image_path)
|
||||
|
||||
data = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(LOCAL_AI_URL, data=json.dumps(data).encode("utf-8")) as resp:
|
||||
if resp.status == 200:
|
||||
result = json.loads(resp.read().decode("utf-8"))
|
||||
return {"success": True, "data": result}
|
||||
else:
|
||||
return {"success": False, "error": f"HTTP {resp.status}"}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
def parse_annotation(response: dict) -> list:
|
||||
"""解析 LocalAI qwen3.5-9b 返回的标注结果(文本格式)"""
|
||||
if not response.get("success"):
|
||||
return []
|
||||
|
||||
content = response.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
|
||||
# 如果没找到目标
|
||||
if "NOT_FOUND" in content or "未找到" in content:
|
||||
return []
|
||||
|
||||
# 解析文本格式:类别 置信度 x1 y1 x2 y2
|
||||
# 例如:pheasant 0.92 0.1 0.2 0.3 0.4
|
||||
parts = content.strip().split()
|
||||
if len(parts) < 5:
|
||||
return []
|
||||
|
||||
cls_name = parts[0].lower()
|
||||
try:
|
||||
confidence = float(parts[1])
|
||||
x1, y1, x2, y2 = [float(v) for v in parts[2:6]]
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
# 确保坐标顺序
|
||||
if x1 > x2 or y1 > y2:
|
||||
x1, x2 = x2, x1
|
||||
y1, y2 = y2, y1
|
||||
|
||||
return [{
|
||||
"class": cls_name,
|
||||
"confidence": confidence,
|
||||
"bbox": [x1, y1, x2, y2],
|
||||
}]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for split in ("train", "val"):
|
||||
for sub in ("images", "labels"):
|
||||
os.makedirs(os.path.join(OUT_DIR, split, sub), exist_ok=True)
|
||||
os.makedirs(PREVIEW_DIR, exist_ok=True)
|
||||
|
||||
for cls in CLASSES:
|
||||
src_dir = os.path.join(IMG_DIR, cls)
|
||||
files = sorted(f for f in os.listdir(src_dir) if f.endswith(".jpg"))
|
||||
print(f"[label] {cls}: {len(files)} 张", flush=True)
|
||||
|
||||
kept = []
|
||||
for f in files:
|
||||
path = os.path.join(src_dir, f)
|
||||
image = Image.open(path).convert("RGB")
|
||||
w, h = image.size
|
||||
|
||||
# 调用 LocalAI 进行标注
|
||||
prompt = ANNOTATION_PROMPT.format(target=cls)
|
||||
result = call_localai(path, prompt)
|
||||
annotations = parse_annotation(result)
|
||||
|
||||
# 取置信度最高的标注
|
||||
best = None
|
||||
best_score = 0
|
||||
for ann in annotations:
|
||||
if ann.get("class") != cls:
|
||||
continue
|
||||
score = ann.get("confidence", 0)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = ann
|
||||
|
||||
if best is None:
|
||||
print(f" {f}: ✗ (未找到目标)", flush=True)
|
||||
continue
|
||||
|
||||
if best_score < MIN_CONF:
|
||||
print(f" {f}: ✗ (置信度 {best_score:.2f} < {MIN_CONF})", flush=True)
|
||||
continue
|
||||
|
||||
x1, y1, x2, y2 = best.get("bbox", [0, 0, 1, 1])
|
||||
kept.append((path, (best_score, x1, y1, x2, y2)))
|
||||
print(f" {f}: ✓ ({best_score:.2f})", flush=True)
|
||||
|
||||
kept.sort(key=lambda t: t[1][0], reverse=True)
|
||||
kept = kept[:KEEP_PER_CLASS]
|
||||
avg = sum(k[1][0] for k in kept) / max(len(kept), 1)
|
||||
print(f" → 保留 {len(kept)} 张(平均置信度 {avg:.2f})", flush=True)
|
||||
|
||||
for i, (path, (score, x1, y1, x2, y2)) in enumerate(kept):
|
||||
split = "train" if i % 10 else "val"
|
||||
dst_img = os.path.join(OUT_DIR, split, "images", f"{cls}_{i:03d}.jpg")
|
||||
shutil.copy(path, dst_img)
|
||||
|
||||
# 计算 YOLO 格式坐标
|
||||
w, h = Image.open(dst_img).size
|
||||
cx = (x1 + x2) / 2 / w
|
||||
cy = (y1 + y2) / 2 / h
|
||||
bw = (x2 - x1) / w
|
||||
bh = (y2 - y1) / h
|
||||
|
||||
label_file = os.path.join(OUT_DIR, split, "labels", f"{cls}_{i:03d}.txt")
|
||||
with open(label_file, "w") as f:
|
||||
f.write(f"{CLASSES.index(cls)} {cx:.6f} {cy:.6f} {bw:.6f} {bh:.6f}\n")
|
||||
|
||||
# 生成预览图
|
||||
img = Image.open(dst_img).convert("RGB")
|
||||
draw = ImageDraw.Draw(img)
|
||||
draw.rectangle([x1, y1, x2, y2], outline="#E53935", width=3)
|
||||
draw.text((x1, max(y1 - 14, 0)), f"{cls} {score:.0%}", fill="#E53935")
|
||||
img.save(os.path.join(PREVIEW_DIR, f"{cls}_{i:03d}.jpg"), "JPEG", quality=85)
|
||||
|
||||
# 生成 data.yaml
|
||||
with open(os.path.join(OUT_DIR, "data.yaml"), "w") as f:
|
||||
f.write(f"path: {OUT_DIR}\n")
|
||||
f.write("train: train/images\nval: val/images\n")
|
||||
f.write(f"names: {CLASSES}\n")
|
||||
print("[done] 数据集就绪:", OUT_DIR, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
为训练图片自动生成标注
|
||||
使用 YOLO 模型检测野鸡并生成 YOLO 格式标注
|
||||
"""
|
||||
|
||||
import cv2
|
||||
import os
|
||||
from pathlib import Path
|
||||
from ultralytics import YOLO
|
||||
|
||||
# 配置
|
||||
MODEL_PATH = Path(__file__).parent / "yolov8s-world.pt"
|
||||
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
|
||||
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
|
||||
|
||||
def create_yolo_structure():
|
||||
"""创建 YOLO 格式目录结构"""
|
||||
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
|
||||
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def convert_to_yolo_format(bbox, img_width, img_height):
|
||||
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
|
||||
x1, y1, x2, y2 = bbox
|
||||
cx = (x1 + x2) / 2 / img_width
|
||||
cy = (y1 + y2) / 2 / img_height
|
||||
w = (x2 - x1) / img_width
|
||||
h = (y2 - y1) / img_height
|
||||
return cx, cy, w, h
|
||||
|
||||
def main():
|
||||
# 加载模型
|
||||
print(f"加载模型: {MODEL_PATH}")
|
||||
model = YOLO(str(MODEL_PATH))
|
||||
|
||||
# 创建目录结构
|
||||
create_yolo_structure()
|
||||
|
||||
# 获取所有图片
|
||||
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
|
||||
image_files = [
|
||||
f for f in IMAGES_DIR.iterdir()
|
||||
if f.suffix.lower() in image_extensions
|
||||
]
|
||||
|
||||
print(f"找到 {len(image_files)} 张图片")
|
||||
|
||||
# 处理每张图片
|
||||
labeled_count = 0
|
||||
for img_path in image_files:
|
||||
print(f"处理: {img_path.name}")
|
||||
|
||||
# 读取图片
|
||||
image = cv2.imread(str(img_path))
|
||||
if image is None:
|
||||
print(f" 跳过: 无法读取 {img_path.name}")
|
||||
continue
|
||||
|
||||
img_height, img_width = image.shape[:2]
|
||||
|
||||
# 推理
|
||||
results = model(image, conf=0.3, iou=0.45)
|
||||
|
||||
# 收集标注
|
||||
labels = []
|
||||
for result in results:
|
||||
boxes = result.boxes
|
||||
if boxes is None or len(boxes) == 0:
|
||||
continue
|
||||
|
||||
for box in boxes:
|
||||
class_name = result.names.get(int(box.cls[0]), "")
|
||||
|
||||
# 只保留 bird 或 pheasant 类别
|
||||
if class_name in ["bird", "pheasant"]:
|
||||
class_id = 0 # 只有一个类别:野鸡
|
||||
bbox = list(map(int, box.xyxy[0].tolist()))
|
||||
conf = float(box.conf[0])
|
||||
|
||||
# 转换为 YOLO 格式
|
||||
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
|
||||
labels.append(f"{class_id} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
|
||||
|
||||
# 保存标注文件
|
||||
if len(labels) > 0:
|
||||
labeled_count += 1
|
||||
# 复制图片
|
||||
dst_img_path = OUTPUT_DIR / "images" / "train" / img_path.name
|
||||
cv2.imwrite(str(dst_img_path), image)
|
||||
|
||||
# 保存标注
|
||||
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
|
||||
with open(label_path, "w") as f:
|
||||
f.write("\n".join(labels))
|
||||
|
||||
print(f" ✓ 标注了 {len(labels)} 个目标")
|
||||
else:
|
||||
print(f" - 未检测到野鸡")
|
||||
|
||||
print(f"\n完成!")
|
||||
print(f" 总图片数: {len(image_files)}")
|
||||
print(f" 有效标注: {labeled_count}")
|
||||
print(f" 输出目录: {OUTPUT_DIR}")
|
||||
|
||||
# 创建 data.yaml
|
||||
data_yaml = OUTPUT_DIR / "data.yaml"
|
||||
with open(data_yaml, "w") as f:
|
||||
f.write(f"""# Observer 数据集配置 - 只识别野鸡
|
||||
path: {OUTPUT_DIR}
|
||||
train: images/train
|
||||
val: images/train
|
||||
|
||||
# 类别
|
||||
nc: 1
|
||||
names: ['pheasant']
|
||||
""")
|
||||
print(f" 数据配置: {data_yaml}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""野鸡图片全自动标注 v2。
|
||||
|
||||
方案:Grounding DINO(开放词汇检测,多提示词)→ NMS 去重 →
|
||||
CLIP 裁剪验证(剔除误检)→ 仍无框时 CLIP 滑动窗口兜底。
|
||||
输出 YOLO 格式标注 + 可视化预览,覆盖 datasets/images/pheasant 下全部图片。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw
|
||||
from torchvision.ops import nms
|
||||
|
||||
from transformers import (
|
||||
GroundingDinoProcessor,
|
||||
GroundingDinoForObjectDetection,
|
||||
CLIPProcessor,
|
||||
CLIPModel,
|
||||
)
|
||||
|
||||
BASE = Path(__file__).parent
|
||||
IMG_DIR = BASE / "datasets" / "images" / "pheasant"
|
||||
OUT_DIR = BASE / "datasets" / "yolo_format"
|
||||
PREVIEW_DIR = BASE / "datasets" / "preview"
|
||||
|
||||
DINO_PROMPTS = [
|
||||
"pheasant",
|
||||
"ring-necked pheasant",
|
||||
"wild pheasant",
|
||||
"common pheasant",
|
||||
"bird",
|
||||
]
|
||||
DINO_BOX_THRESHOLD = 0.13
|
||||
DINO_TEXT_THRESHOLD = 0.15
|
||||
NMS_IOU = 0.45
|
||||
KEEP_THRESHOLD = 0.16 # 低于此分的框必须通过 CLIP 验证才保留
|
||||
MIN_BOX_AREA = 0.002 # 框面积占比下限(排除过小误检)
|
||||
|
||||
CLIP_POSITIVE = ["a photo of a pheasant", "a photo of a wild bird"]
|
||||
CLIP_NEGATIVE = [
|
||||
"a photo of grass and leaves",
|
||||
"a photo of rocks and soil",
|
||||
"a photo of a landscape",
|
||||
"a photo of a fence",
|
||||
"a photo of trees",
|
||||
]
|
||||
CLIP_VERIFY_THRESHOLD = 0.40 # softmax(正例) >= 此值才算验证通过
|
||||
FALLBACK_WINDOW_SIZES = [0.5, 0.35, 0.25] # 滑动窗口占图宽比例
|
||||
|
||||
|
||||
class AutoLabeler:
|
||||
def __init__(self):
|
||||
print("加载 Grounding DINO...", flush=True)
|
||||
self.proc = GroundingDinoProcessor.from_pretrained("IDEA-Research/grounding-dino-tiny")
|
||||
self.dino = GroundingDinoForObjectDetection.from_pretrained("IDEA-Research/grounding-dino-tiny")
|
||||
self.dino.eval()
|
||||
|
||||
print("加载 CLIP...", flush=True)
|
||||
self.clip_proc = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
|
||||
self.clip = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
|
||||
self.clip.eval()
|
||||
|
||||
pos_tokens = self.clip_proc(text=CLIP_POSITIVE + CLIP_NEGATIVE, return_tensors="pt", padding=True)
|
||||
with torch.no_grad():
|
||||
out = self.clip.get_text_features(**pos_tokens)
|
||||
if hasattr(out, "text_embeds"):
|
||||
feats = out.text_embeds
|
||||
elif hasattr(out, "pooler_output"):
|
||||
feats = out.pooler_output
|
||||
else:
|
||||
feats = out
|
||||
feats = feats / feats.norm(dim=-1, keepdim=True)
|
||||
self.pos_feats = feats[: len(CLIP_POSITIVE)]
|
||||
self.neg_feats = feats[len(CLIP_POSITIVE):]
|
||||
|
||||
def clip_score(self, region: Image.Image) -> float:
|
||||
"""返回裁剪区域是野鸡的概率(softmax 归一化,0~1)"""
|
||||
inputs = self.clip_proc(images=region, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
out = self.clip.get_image_features(**inputs)
|
||||
if hasattr(out, "pooler_output"):
|
||||
img_feat = out.pooler_output
|
||||
else:
|
||||
img_feat = out
|
||||
img_feat = img_feat / img_feat.norm(dim=-1, keepdim=True)
|
||||
sim_pos = img_feat @ self.pos_feats.T # (1, P)
|
||||
sim_neg = img_feat @ self.neg_feats.T # (1, N)
|
||||
logits = torch.cat([sim_pos * 100, sim_neg * 100], dim=1)
|
||||
prob = logits.softmax(dim=1)[0, : len(CLIP_POSITIVE)].max().item()
|
||||
return prob
|
||||
|
||||
def dino_detect(self, img: Image.Image):
|
||||
"""Grounding DINO 检测,返回 [x1,y1,x2,y2] 绝对坐标列表"""
|
||||
inputs = self.proc(images=img, text=DINO_PROMPTS, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
out = self.dino(**inputs)
|
||||
res = self.proc.post_process_grounded_object_detection(
|
||||
out, input_ids=inputs["input_ids"],
|
||||
threshold=DINO_BOX_THRESHOLD, text_threshold=DINO_TEXT_THRESHOLD,
|
||||
target_sizes=[img.size[::-1]],
|
||||
)[0]
|
||||
w, h = img.size
|
||||
boxes = res["boxes"].tolist()
|
||||
scores = res["scores"].tolist()
|
||||
valid = []
|
||||
for (x1, y1, x2, y2), s in zip(boxes, scores):
|
||||
bw, bh = (x2 - x1) / w, (y2 - y1) / h
|
||||
if bw * bh < MIN_BOX_AREA:
|
||||
continue
|
||||
valid.append((x1, y1, x2, y2, s))
|
||||
return valid
|
||||
|
||||
def clip_fallback(self, img: Image.Image):
|
||||
"""滑动窗口找最像野鸡的区域(兜底),返回 [x1,y1,x2,y2] 或 None"""
|
||||
w, h = img.size
|
||||
base = 768
|
||||
scale = base / max(w, h)
|
||||
small = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
|
||||
sw, sh = small.size
|
||||
best = None
|
||||
best_score = 0.0
|
||||
for f in FALLBACK_WINDOW_SIZES:
|
||||
ws, hs = int(sw * f), int(sh * f)
|
||||
stride = max(int(ws * 0.5), 8)
|
||||
for y in range(0, max(sh - hs, 1), stride):
|
||||
for x in range(0, max(sw - ws, 1), stride):
|
||||
region = small.crop((x, y, x + ws, y + hs))
|
||||
s = self.clip_score(region)
|
||||
if s > best_score:
|
||||
best_score = s
|
||||
best = (x, y, x + ws, y + hs)
|
||||
if best is None:
|
||||
return None, best_score
|
||||
x1, y1, x2, y2 = [v / scale for v in best]
|
||||
return (x1, y1, x2, y2), best_score
|
||||
|
||||
def label_image(self, img: Image.Image, name: str):
|
||||
"""返回 (boxes, source, debug),boxes 为 [x1,y1,x2,y2] 绝对坐标列表"""
|
||||
w, h = img.size
|
||||
|
||||
cands = self.dino_detect(img)
|
||||
if cands:
|
||||
boxes = torch.tensor([[c[0], c[1], c[2], c[3]] for c in cands], dtype=torch.float32)
|
||||
scores = torch.tensor([c[4] for c in cands], dtype=torch.float32)
|
||||
keep = nms(boxes, scores, NMS_IOU)
|
||||
kept = [(cands[i][0], cands[i][1], cands[i][2], cands[i][3], cands[i][4]) for i in keep]
|
||||
|
||||
# CLIP 验证每个框
|
||||
final = []
|
||||
for x1, y1, x2, y2, s in kept:
|
||||
pad_x, pad_y = (x2 - x1) * 0.2, (y2 - y1) * 0.2
|
||||
region = img.crop((max(x1 - pad_x, 0), max(y1 - pad_y, 0),
|
||||
min(x2 + pad_x, w), min(y2 + pad_y, h)))
|
||||
prob = self.clip_score(region)
|
||||
final.append((x1, y1, x2, y2, s, prob))
|
||||
best_clip = max(p for _, _, _, _, _, p in final)
|
||||
if best_clip < CLIP_VERIFY_THRESHOLD:
|
||||
print(f" {name}: 整图框分均过低,改用 CLIP 兜底 (best_clip={best_clip:.2f})", flush=True)
|
||||
box, score = self.clip_fallback(img)
|
||||
if box is not None:
|
||||
x1, y1, x2, y2 = box
|
||||
return [(x1, y1, x2, y2, score, f"clip({score:.2f})")], "clip"
|
||||
return [], "none"
|
||||
kept_final = [f for f in final if f[5] >= 0.20]
|
||||
if not kept_final:
|
||||
kept_final = [max(final, key=lambda t: t[5])]
|
||||
for x1, y1, x2, y2, s, p in final:
|
||||
if (x1, y1, x2, y2, s, p) not in kept_final:
|
||||
print(f" {name}: 剔除低分框 clip={p:.2f} [{x1:.0f},{y1:.0f},{x2:.0f},{y2:.0f}]", flush=True)
|
||||
return [(x1, y1, x2, y2, s, f"dino(clip={p:.2f})")
|
||||
for x1, y1, x2, y2, s, p in kept_final], "dino"
|
||||
else:
|
||||
print(f" {name}: DINO 无候选,使用 CLIP 兜底", flush=True)
|
||||
|
||||
# 兜底:CLIP 滑动窗口
|
||||
box, score = self.clip_fallback(img)
|
||||
if box is not None:
|
||||
x1, y1, x2, y2 = box
|
||||
return [(x1, y1, x2, y2, score, f"clip({score:.2f})")], "clip"
|
||||
return [], "none"
|
||||
|
||||
|
||||
def main():
|
||||
(OUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
|
||||
(OUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
|
||||
PREVIEW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
labeler = AutoLabeler()
|
||||
|
||||
files = sorted(f for f in IMG_DIR.iterdir() if f.suffix.lower() in {".png", ".jpg", ".jpeg"})
|
||||
print(f"共 {len(files)} 张图片", flush=True)
|
||||
|
||||
summary = []
|
||||
for p in files:
|
||||
img = Image.open(p).convert("RGB")
|
||||
w, h = img.size
|
||||
boxes, src = labeler.label_image(img, p.name)
|
||||
summary.append((p.name, len(boxes), src))
|
||||
|
||||
label_path = OUT_DIR / "labels" / "train" / (p.stem + ".txt")
|
||||
img.save(OUT_DIR / "images" / "train" / p.name)
|
||||
if boxes:
|
||||
with open(label_path, "w") as f:
|
||||
for x1, y1, x2, y2, s, _ in boxes:
|
||||
cx, cy = (x1 + x2) / 2 / w, (y1 + y2) / 2 / h
|
||||
bw, bh = (x2 - x1) / w, (y2 - y1) / h
|
||||
f.write(f"0 {cx:.6f} {cy:.6f} {bw:.6f} {bh:.6f}\n")
|
||||
|
||||
# 预览图
|
||||
draw = ImageDraw.Draw(img)
|
||||
for x1, y1, x2, y2, s, tag in boxes:
|
||||
draw.rectangle([x1, y1, x2, y2], outline="#E53935", width=4)
|
||||
draw.text((x1 + 4, max(y1 - 18, 0)), f"pheasant {s:.2f} [{tag}]", fill="#E53935")
|
||||
img.save(PREVIEW_DIR / (p.stem + ".jpg"), "JPEG", quality=85)
|
||||
|
||||
status = "✓" if boxes else "✗"
|
||||
print(f"{status} {p.name}: {len(boxes)} 框 ({src})", flush=True)
|
||||
|
||||
# 生成 data.yaml
|
||||
with open(OUT_DIR / "data.yaml", "w") as f:
|
||||
f.write(f"path: {OUT_DIR}\n")
|
||||
f.write("train: images/train\nval: images/train\n")
|
||||
f.write("nc: 1\nnames: ['pheasant']\n")
|
||||
|
||||
labeled = sum(1 for _, n, _ in summary if n > 0)
|
||||
print(f"\n完成: {labeled}/{len(summary)} 张已标注", flush=True)
|
||||
for name, n, src in summary:
|
||||
print(f" {name}: {n} 框 ({src})", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
使用 CLIP 模型的滑动窗口方法检测野鸡位置
|
||||
生成 YOLO 格式的标注文件
|
||||
"""
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
import clip
|
||||
from PIL import Image
|
||||
from pathlib import Path
|
||||
from torchvision.ops import nms
|
||||
|
||||
# 配置
|
||||
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
|
||||
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
|
||||
|
||||
# 检测配置
|
||||
WINDOW_SIZES = [128, 256, 512] # 滑动窗口大小(增大)
|
||||
STRIDE_RATIO = 0.7 # 窗口滑动步长比例(增大)
|
||||
CONFIDENCE_THRESHOLD = 0.5 # 置信度阈值(提高)
|
||||
NMS_THRESHOLD = 0.5 # NMS 阈值(提高)
|
||||
|
||||
# 野鸡的文本描述
|
||||
PHEASANT_PROMPTS = [
|
||||
"a photo of a pheasant in the wild",
|
||||
"a wild pheasant in natural habitat",
|
||||
"a bird with colorful feathers in grass",
|
||||
]
|
||||
|
||||
class CLIPDetector:
|
||||
def __init__(self):
|
||||
print("加载 CLIP 模型...")
|
||||
self.model, self.preprocess = clip.load("ViT-B/32", device="cpu")
|
||||
self.model.eval()
|
||||
print("CLIP 模型加载完成")
|
||||
|
||||
# 预计算文本特征
|
||||
print("预计算文本特征...")
|
||||
text_tokens = clip.tokenize(PHEASANT_PROMPTS).to("cpu")
|
||||
with torch.no_grad():
|
||||
self.text_features = self.model.encode_text(text_tokens)
|
||||
self.text_features /= self.text_features.norm(dim=-1, keepdim=True)
|
||||
|
||||
def classify_region(self, image_region):
|
||||
"""对图像区域进行分类"""
|
||||
# 转换为 PIL Image
|
||||
if isinstance(image_region, np.ndarray):
|
||||
image_region = Image.fromarray(cv2.cvtColor(image_region, cv2.COLOR_BGR2RGB))
|
||||
|
||||
# 预处理
|
||||
image_input = self.preprocess(image_region).unsqueeze(0).to("cpu")
|
||||
|
||||
# 计算图像特征
|
||||
with torch.no_grad():
|
||||
image_features = self.model.encode_image(image_input)
|
||||
image_features /= image_features.norm(dim=-1, keepdim=True)
|
||||
|
||||
# 计算相似度
|
||||
similarity = (100.0 * image_features @ self.text_features.T).softmax(dim=-1)
|
||||
|
||||
# 返回最高分
|
||||
return float(similarity[0].max())
|
||||
|
||||
def detect(self, image):
|
||||
"""检测图片中的野鸡"""
|
||||
h, w = image.shape[:2]
|
||||
detections = []
|
||||
|
||||
# 多尺度滑动窗口
|
||||
for window_size in WINDOW_SIZES:
|
||||
stride = int(window_size * STRIDE_RATIO)
|
||||
|
||||
# 滑动窗口
|
||||
for y in range(0, h - window_size, stride):
|
||||
for x in range(0, w - window_size, stride):
|
||||
# 提取窗口区域
|
||||
region = image[y:y+window_size, x:x+window_size]
|
||||
|
||||
# 分类
|
||||
score = self.classify_region(region)
|
||||
|
||||
# 如果置信度足够高,保存检测结果
|
||||
if score > CONFIDENCE_THRESHOLD:
|
||||
detections.append({
|
||||
'bbox': [x, y, x + window_size, y + window_size],
|
||||
'score': score,
|
||||
})
|
||||
|
||||
# NMS 去重
|
||||
if len(detections) > 0:
|
||||
detections = self.nms(detections)
|
||||
|
||||
return detections
|
||||
|
||||
def nms(self, detections):
|
||||
"""非极大值抑制"""
|
||||
if len(detections) == 0:
|
||||
return []
|
||||
|
||||
# 转换为 torch 格式
|
||||
boxes = torch.tensor([d['bbox'] for d in detections], dtype=torch.float32)
|
||||
scores = torch.tensor([d['score'] for d in detections], dtype=torch.float32)
|
||||
|
||||
# 应用 NMS
|
||||
keep_indices = nms(boxes, scores, NMS_THRESHOLD)
|
||||
|
||||
# 保留 NMS 后的检测结果
|
||||
filtered_detections = [detections[i] for i in keep_indices]
|
||||
|
||||
return filtered_detections
|
||||
|
||||
def convert_to_yolo_format(bbox, img_width, img_height):
|
||||
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
|
||||
x1, y1, x2, y2 = bbox
|
||||
cx = (x1 + x2) / 2 / img_width
|
||||
cy = (y1 + y2) / 2 / img_height
|
||||
w = (x2 - x1) / img_width
|
||||
h = (y2 - y1) / img_height
|
||||
return cx, cy, w, h
|
||||
|
||||
def main():
|
||||
# 初始化检测器
|
||||
detector = CLIPDetector()
|
||||
|
||||
# 创建输出目录
|
||||
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
|
||||
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 获取所有图片
|
||||
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
|
||||
image_files = [
|
||||
f for f in IMAGES_DIR.iterdir()
|
||||
if f.suffix.lower() in image_extensions
|
||||
]
|
||||
|
||||
print(f"找到 {len(image_files)} 张图片")
|
||||
|
||||
# 处理每张图片
|
||||
labeled_count = 0
|
||||
for img_path in image_files:
|
||||
print(f"\n处理: {img_path.name}")
|
||||
|
||||
# 读取图片
|
||||
image = cv2.imread(str(img_path))
|
||||
if image is None:
|
||||
print(f" 跳过: 无法读取 {img_path.name}")
|
||||
continue
|
||||
|
||||
img_height, img_width = image.shape[:2]
|
||||
|
||||
# 检测野鸡
|
||||
detections = detector.detect(image)
|
||||
|
||||
# 生成标注
|
||||
labels = []
|
||||
for det in detections:
|
||||
bbox = det['bbox']
|
||||
score = det['score']
|
||||
|
||||
# 转换为 YOLO 格式
|
||||
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
|
||||
labels.append(f"0 {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
|
||||
|
||||
# 在图片上绘制检测框(用于可视化)
|
||||
x1, y1, x2, y2 = bbox
|
||||
cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
|
||||
cv2.putText(image, f"pheasant: {score:.2f}", (int(x1), int(y1) - 10),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
|
||||
|
||||
# 保存标注文件
|
||||
if len(labels) > 0:
|
||||
labeled_count += 1
|
||||
# 保存带标注的图片(用于可视化)
|
||||
cv2.imwrite(str(OUTPUT_DIR / "images" / "train" / img_path.name), image)
|
||||
|
||||
# 保存标注
|
||||
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
|
||||
with open(label_path, "w") as f:
|
||||
f.writelines(labels)
|
||||
|
||||
print(f" ✓ 标注了 {len(labels)} 个目标")
|
||||
else:
|
||||
print(f" - 未检测到野鸡")
|
||||
|
||||
print(f"\n完成!")
|
||||
print(f" 总图片数: {len(image_files)}")
|
||||
print(f" 有效标注: {labeled_count}")
|
||||
print(f" 输出目录: {OUTPUT_DIR}")
|
||||
|
||||
# 创建 data.yaml
|
||||
data_yaml = OUTPUT_DIR / "data.yaml"
|
||||
with open(data_yaml, "w") as f:
|
||||
f.write(f"""# Observer 数据集配置 - 只识别野鸡
|
||||
path: {OUTPUT_DIR}
|
||||
train: images/train
|
||||
val: images/train
|
||||
|
||||
# 类别
|
||||
nc: 1
|
||||
names: ['pheasant']
|
||||
""")
|
||||
print(f" 数据配置: {data_yaml}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,300 @@
|
||||
"""从 Wikimedia Commons 下载训练图片(自由版权,可离线使用)。
|
||||
|
||||
策略:优先用物种分类目录(图片内容精确),再用全文搜索补充场景/姿态/光线多样性。
|
||||
输出: datasets/images/<class>/<index>.jpg
|
||||
串行 + 失败重试 + pHash 视觉去重。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
import imagehash
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
API = "https://commons.wikimedia.org/w/api.php"
|
||||
OUT = os.path.join(os.path.dirname(__file__), "datasets", "images")
|
||||
|
||||
# 更多图片来源
|
||||
ADDITIONAL_SOURCES = {
|
||||
"inaturalist": "https://api.inaturalist.org/v1/observations",
|
||||
"flickr": "https://api.flickr.com/services/rest/",
|
||||
}
|
||||
|
||||
SOURCES = {
|
||||
"pheasant": {
|
||||
"categories": ["Phasianus colchicus"],
|
||||
"queries": [
|
||||
# 户外真实场景(重点!)
|
||||
"pheasant in field",
|
||||
"pheasant in grassland",
|
||||
"pheasant in meadow",
|
||||
"pheasant in farmland",
|
||||
"pheasant in countryside",
|
||||
"pheasant in wild",
|
||||
# 部分遮挡场景(重点!实际使用场景)
|
||||
"pheasant hiding in grass",
|
||||
"pheasant hiding in bushes",
|
||||
"pheasant partially hidden",
|
||||
"pheasant behind vegetation",
|
||||
"pheasant peeking through grass",
|
||||
"pheasant concealed in foliage",
|
||||
"pheasant camouflaged",
|
||||
"pheasant blending in",
|
||||
# 不同距离和角度
|
||||
"pheasant distant view",
|
||||
"pheasant far away",
|
||||
"pheasant small in frame",
|
||||
"pheasant side view",
|
||||
"pheasant back view",
|
||||
"pheasant from behind",
|
||||
# 不同姿态和行为
|
||||
"pheasant walking in grass",
|
||||
"pheasant foraging",
|
||||
"pheasant feeding",
|
||||
"pheasant running",
|
||||
"pheasant flying low",
|
||||
# 光线条件(户外真实光线)
|
||||
"pheasant natural light",
|
||||
"pheasant daylight",
|
||||
"pheasant shade",
|
||||
"pheasant shadow",
|
||||
"pheasant backlit",
|
||||
"pheasant overcast",
|
||||
# 季节环境
|
||||
"pheasant in autumn",
|
||||
"pheasant in winter",
|
||||
"pheasant in spring",
|
||||
"pheasant in summer",
|
||||
"pheasant in dry grass",
|
||||
"pheasant in green grass",
|
||||
"pheasant in snow",
|
||||
"pheasant in mud",
|
||||
],
|
||||
},
|
||||
}
|
||||
PER_CLASS = 80
|
||||
MIN_PIXEL = 320
|
||||
RETRY = 2
|
||||
WORKERS = 1
|
||||
SLEEP_S = 2.0
|
||||
RATE_LIMIT_WAIT_S = 60
|
||||
|
||||
|
||||
def api_request(params: dict) -> dict:
|
||||
url = API + "?" + urllib.parse.urlencode(params)
|
||||
for attempt in range(RETRY + 1):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "observer-training/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
except Exception:
|
||||
if attempt == RETRY:
|
||||
raise
|
||||
time.sleep(2 * (attempt + 1))
|
||||
|
||||
|
||||
def collect_pages(pages: dict, results: list) -> None:
|
||||
for p in pages.values():
|
||||
info = (p.get("imageinfo") or [{}])[0]
|
||||
thumb = info.get("thumburl")
|
||||
if not thumb:
|
||||
continue
|
||||
w, h = info.get("width", 0), info.get("height", 0)
|
||||
if min(w, h) < MIN_PIXEL:
|
||||
continue
|
||||
results.append({"url": thumb, "w": w, "h": h, "title": p.get("title", "")})
|
||||
|
||||
|
||||
def category_images(category: str, limit: int) -> list[dict]:
|
||||
results = []
|
||||
params = {
|
||||
"action": "query",
|
||||
"generator": "categorymembers",
|
||||
"gcmtitle": f"Category:{category}",
|
||||
"gcmtype": "file",
|
||||
"gcmlimit": "50",
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url|size",
|
||||
"iiurlwidth": "640",
|
||||
"format": "json",
|
||||
}
|
||||
while len(results) < limit:
|
||||
data = api_request(params)
|
||||
pages = (data.get("query", {}) or {}).get("pages", {})
|
||||
collect_pages(pages, results)
|
||||
cont = (data.get("continue") or {}).get("gcmcontinue")
|
||||
if not cont:
|
||||
break
|
||||
params["gcmcontinue"] = cont
|
||||
return results[:limit]
|
||||
|
||||
|
||||
def search_images(query: str, limit: int) -> list[dict]:
|
||||
results = []
|
||||
params = {
|
||||
"action": "query",
|
||||
"generator": "search",
|
||||
"gsrsearch": f"filetype:bitmap {query}",
|
||||
"gsrnamespace": "6",
|
||||
"gsrlimit": str(limit),
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url|size",
|
||||
"iiurlwidth": "640",
|
||||
"format": "json",
|
||||
}
|
||||
data = api_request(params)
|
||||
pages = (data.get("query", {}) or {}).get("pages", {})
|
||||
collect_pages(pages, results)
|
||||
return results
|
||||
|
||||
|
||||
def is_text_heavy(image_path: str) -> bool:
|
||||
"""检测图片是否包含大量文字或为图表/标志"""
|
||||
try:
|
||||
img = Image.open(image_path).convert("L") # 灰度
|
||||
arr = np.array(img, dtype=np.float32)
|
||||
|
||||
# 1. 检测边缘密度(文字产生大量边缘)
|
||||
# 简单边缘检测:计算像素梯度
|
||||
dx = np.abs(np.diff(arr, axis=1))
|
||||
dy = np.abs(np.diff(arr, axis=0))
|
||||
edge_density = (np.mean(dx) + np.mean(dy)) / 2
|
||||
|
||||
# 2. 检测颜色方差(图表通常颜色单一)
|
||||
# 转回RGB检查
|
||||
img_rgb = Image.open(image_path).convert("RGB")
|
||||
arr_rgb = np.array(img_rgb, dtype=np.float32)
|
||||
color_variance = np.std(arr_rgb)
|
||||
|
||||
# 3. 检测对比度(文字通常有高对比度的边缘)
|
||||
contrast = np.std(arr)
|
||||
|
||||
# 判断逻辑:
|
||||
# - 高边缘密度 + 低颜色方差 = 可能是图表/标志
|
||||
# - 高对比度 + 高边缘密度 = 可能是文字图片
|
||||
if edge_density > 30 and color_variance < 50:
|
||||
return True
|
||||
if edge_density > 40 and contrast > 80:
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def download(url: str, path: str) -> bool:
|
||||
for attempt in range(RETRY + 1):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "observer-training/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
data = r.read()
|
||||
if "image/" not in r.headers.get("Content-Type", ""):
|
||||
return False
|
||||
if len(data) < 10_000:
|
||||
return False
|
||||
with open(path, "wb") as f:
|
||||
f.write(data)
|
||||
return True
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
# 限流:冷却后重试
|
||||
print(f" ⏳ 限流(429),冷却 {RATE_LIMIT_WAIT_S}s", flush=True)
|
||||
time.sleep(RATE_LIMIT_WAIT_S)
|
||||
continue
|
||||
if attempt == RETRY:
|
||||
return False
|
||||
time.sleep(2 * (attempt + 1))
|
||||
except Exception:
|
||||
if attempt == RETRY:
|
||||
return False
|
||||
time.sleep(2 * (attempt + 1))
|
||||
|
||||
|
||||
def main():
|
||||
for cls, src in SOURCES.items():
|
||||
cls_dir = os.path.join(OUT, cls)
|
||||
os.makedirs(cls_dir, exist_ok=True)
|
||||
existing = len([f for f in os.listdir(cls_dir) if f.endswith(".jpg")])
|
||||
if existing >= PER_CLASS:
|
||||
print(f"[skip] {cls}: 已有 {existing} 张", flush=True)
|
||||
continue
|
||||
|
||||
# 重建去重池(续跑兼容)
|
||||
seen_hashes = []
|
||||
for f in os.listdir(cls_dir):
|
||||
if f.endswith(".jpg"):
|
||||
try:
|
||||
seen_hashes.append(imagehash.phash(Image.open(os.path.join(cls_dir, f))))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
candidates = []
|
||||
seen_titles = set()
|
||||
for cat in src["categories"]:
|
||||
print(f"[category] {cls} <- {cat}", flush=True)
|
||||
try:
|
||||
for item in category_images(cat, 80):
|
||||
if item["title"] in seen_titles:
|
||||
continue
|
||||
seen_titles.add(item["title"])
|
||||
candidates.append(item)
|
||||
except Exception as e:
|
||||
print(f" ! category 失败: {e}", flush=True)
|
||||
for query in src["queries"]:
|
||||
if len(candidates) >= PER_CLASS * 2:
|
||||
break
|
||||
print(f"[fetch] {cls} <- \"{query}\"", flush=True)
|
||||
try:
|
||||
for item in search_images(query, 80):
|
||||
if item["title"] in seen_titles:
|
||||
continue
|
||||
seen_titles.add(item["title"])
|
||||
candidates.append(item)
|
||||
except Exception as e:
|
||||
print(f" ! 搜索失败: {e}", flush=True)
|
||||
print(f"[download] {cls}: 候选 {len(candidates)} 张", flush=True)
|
||||
|
||||
saved = existing
|
||||
fail = 0
|
||||
dup = 0
|
||||
for item in candidates:
|
||||
if saved >= PER_CLASS:
|
||||
break
|
||||
saved += 1
|
||||
path = os.path.join(cls_dir, f"{saved:03d}.jpg")
|
||||
if not download(item["url"], path):
|
||||
fail += 1
|
||||
saved -= 1
|
||||
continue
|
||||
|
||||
# 过滤文字/图表类图片
|
||||
if is_text_heavy(path):
|
||||
os.remove(path)
|
||||
saved -= 1
|
||||
continue
|
||||
|
||||
try:
|
||||
h = imagehash.phash(Image.open(path))
|
||||
if any(h - other <= 8 for other in seen_hashes):
|
||||
os.remove(path)
|
||||
dup += 1
|
||||
saved -= 1
|
||||
continue
|
||||
seen_hashes.append(h)
|
||||
except Exception:
|
||||
pass
|
||||
if saved % 10 == 0:
|
||||
print(f" + {cls}: {saved}", flush=True)
|
||||
time.sleep(SLEEP_S)
|
||||
print(f"[done] {cls}: {saved} 张(失败 {fail},重复 {dup})", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
"""用 LocalAI qwen3.5-9b 过滤不含活体动物的图片。
|
||||
|
||||
用法:python filter_images.py
|
||||
输入:datasets/images/<class>/*.jpg
|
||||
输出:删除不含目标动物的图片
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import base64
|
||||
import sys
|
||||
import urllib.request
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
# LocalAI 服务器配置
|
||||
LOCAL_AI_URL = "http://192.168.3.210:18080/v1/chat/completions"
|
||||
MODEL_NAME = "qwen3.5-9b"
|
||||
|
||||
# 过滤配置
|
||||
BASE = os.path.dirname(__file__)
|
||||
IMG_DIR = os.path.join(BASE, "datasets", "images")
|
||||
CLASSES = ["pheasant"]
|
||||
MAX_IMAGE_SIZE = 400 # 缩放到最大边长
|
||||
|
||||
# 过滤提示词
|
||||
FILTER_PROMPT = """你是一个图片质量检查助手。请判断这张图片是否包含【{target}】的活体动物照片。
|
||||
|
||||
判断标准:
|
||||
✓ 保留:真实动物照片(活体、自然姿态、野外或自然环境)
|
||||
✗ 删除:标本照片、插画、图表、文字图片、logo、标志、空场景、纯风景
|
||||
|
||||
只回答:KEEP 或 DELETE"""
|
||||
|
||||
|
||||
def encode_image_to_base64(path: str) -> str:
|
||||
"""读取图片并压缩后转为 base64"""
|
||||
img = Image.open(path).convert("RGB") # 转为 RGB 避免 RGBA 问题
|
||||
# 缩放图片以减少 API 负载
|
||||
img.thumbnail((MAX_IMAGE_SIZE, MAX_IMAGE_SIZE), Image.Resampling.LANCZOS)
|
||||
buffer = BytesIO()
|
||||
img.save(buffer, format="JPEG", quality=70)
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
def call_localai(image_path: str, prompt: str) -> str:
|
||||
"""调用 LocalAI 检测图片是否包含活体动物"""
|
||||
image_b64 = encode_image_to_base64(image_path)
|
||||
|
||||
data = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0.1,
|
||||
}
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(LOCAL_AI_URL, data=json.dumps(data).encode("utf-8"), timeout=60) as resp:
|
||||
if resp.status == 200:
|
||||
result = json.loads(resp.read().decode("utf-8"))
|
||||
return result.get("choices", [{}])[0].get("message", {}).get("content", "").strip().upper()
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
import time
|
||||
time.sleep(2 * (attempt + 1))
|
||||
else:
|
||||
print(f" ! API 错误: {e}", flush=True)
|
||||
return "ERROR"
|
||||
|
||||
|
||||
def main():
|
||||
total_deleted = 0
|
||||
|
||||
for cls in CLASSES:
|
||||
cls_dir = os.path.join(IMG_DIR, cls)
|
||||
if not os.path.exists(cls_dir):
|
||||
continue
|
||||
|
||||
files = sorted(f for f in os.listdir(cls_dir) if f.endswith(".jpg"))
|
||||
print(f"\n[filter] {cls}: {len(files)} 张", flush=True)
|
||||
|
||||
deleted = 0
|
||||
kept = 0
|
||||
|
||||
for f in files:
|
||||
path = os.path.join(cls_dir, f)
|
||||
prompt = FILTER_PROMPT.format(target=cls)
|
||||
result = call_localai(path, prompt)
|
||||
|
||||
if "DELETE" in result:
|
||||
os.remove(path)
|
||||
deleted += 1
|
||||
print(f" {f}: DELETE", flush=True)
|
||||
else:
|
||||
kept += 1
|
||||
print(f" {f}: KEEP", flush=True)
|
||||
|
||||
print(f" → 保留 {kept} 张,删除 {deleted} 张", flush=True)
|
||||
total_deleted += deleted
|
||||
|
||||
print(f"\n[done] 共删除 {total_deleted} 张图片", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
"""用本地 CLIP 模型过滤不含活体动物的图片。
|
||||
|
||||
用法:python filter_images_clip.py
|
||||
输入:datasets/images/<class>/*.jpg
|
||||
输出:删除不含目标动物的图片
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
import clip
|
||||
from PIL import Image
|
||||
|
||||
# 过滤配置
|
||||
BASE = os.path.dirname(__file__)
|
||||
IMG_DIR = os.path.join(BASE, "datasets", "images")
|
||||
CLASSES = ["pheasant"]
|
||||
MODEL_PATH = os.path.join(BASE, "..", "weights", "clip", "ViT-B-32.pt")
|
||||
|
||||
# 判断阈值 - 相似度低于此值的图片将被删除
|
||||
SIMILARITY_THRESHOLD = 0.10
|
||||
|
||||
# 每个类别的正向和负向描述
|
||||
PROMPTS = {
|
||||
"pheasant": {
|
||||
"positive": [
|
||||
"a photo of a pheasant in the wild",
|
||||
"a wild pheasant in natural outdoor habitat",
|
||||
"a pheasant walking in grass or field",
|
||||
"a pheasant hiding in bushes",
|
||||
"a pheasant in natural environment",
|
||||
],
|
||||
"negative": [
|
||||
"a taxidermy pheasant",
|
||||
"an illustration of a pheasant",
|
||||
"a drawing of a bird",
|
||||
"a painting of a bird",
|
||||
"a cartoon of a bird",
|
||||
"a person holding a bird",
|
||||
"a person catching a bird",
|
||||
"a bird in a cage",
|
||||
"a bird indoors",
|
||||
"a bird in a house",
|
||||
"a bird in a zoo",
|
||||
"a bird in captivity",
|
||||
"a logo or icon",
|
||||
"text or writing",
|
||||
"a landscape without animals",
|
||||
"a statue or sculpture",
|
||||
"a stuffed animal",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
# 加载 CLIP 模型
|
||||
print("[init] 加载 CLIP 模型...", flush=True)
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model, preprocess = clip.load("ViT-B/32", device=device, download_root=os.path.join(BASE, "..", "weights"))
|
||||
|
||||
# 缓存文本特征
|
||||
text_features_cache = {}
|
||||
for cls in CLASSES:
|
||||
if cls not in PROMPTS:
|
||||
continue
|
||||
|
||||
pos_texts = clip.tokenize(PROMPTS[cls]["positive"]).to(device)
|
||||
neg_texts = clip.tokenize(PROMPTS[cls]["negative"]).to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
pos_features = model.encode_text(pos_texts)
|
||||
neg_features = model.encode_text(neg_texts)
|
||||
# 取正向描述的平均特征
|
||||
pos_features = pos_features.mean(dim=0, keepdim=True)
|
||||
pos_features /= pos_features.norm(dim=-1, keepdim=True)
|
||||
neg_features = neg_features.mean(dim=0, keepdim=True)
|
||||
neg_features /= neg_features.norm(dim=-1, keepdim=True)
|
||||
|
||||
text_features_cache[cls] = (pos_features, neg_features)
|
||||
|
||||
total_deleted = 0
|
||||
|
||||
for cls in CLASSES:
|
||||
cls_dir = os.path.join(IMG_DIR, cls)
|
||||
if not os.path.exists(cls_dir):
|
||||
continue
|
||||
|
||||
files = sorted(f for f in os.listdir(cls_dir) if f.endswith(".jpg"))
|
||||
print(f"\n[filter] {cls}: {len(files)} 张", flush=True)
|
||||
|
||||
deleted = 0
|
||||
kept = 0
|
||||
pos_features, neg_features = text_features_cache[cls]
|
||||
|
||||
for f in files:
|
||||
path = os.path.join(cls_dir, f)
|
||||
try:
|
||||
image = preprocess(Image.open(path).convert("RGB")).unsqueeze(0).to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
image_features = model.encode_image(image)
|
||||
image_features /= image_features.norm(dim=-1, keepdim=True)
|
||||
|
||||
# 计算与正向和负向描述的相似度
|
||||
pos_similarity = (image_features @ pos_features.T).item()
|
||||
neg_similarity = (image_features @ neg_features.T).item()
|
||||
|
||||
# 综合得分:正向相似度 - 负向相似度
|
||||
score = pos_similarity - neg_similarity
|
||||
|
||||
if score < SIMILARITY_THRESHOLD:
|
||||
os.remove(path)
|
||||
deleted += 1
|
||||
print(f" {f}: DELETE (score={score:.3f})", flush=True)
|
||||
else:
|
||||
kept += 1
|
||||
print(f" {f}: KEEP (score={score:.3f})", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f" {f}: ERROR ({e})", flush=True)
|
||||
kept += 1 # 出错时保留
|
||||
|
||||
print(f" → 保留 {kept} 张,删除 {deleted} 张", flush=True)
|
||||
total_deleted += deleted
|
||||
|
||||
print(f"\n[done] 共删除 {total_deleted} 张图片", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
训练 YOLOv8n 模型 - 只识别野鸡(pheasant)
|
||||
使用现有的训练数据进行迁移学习
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from ultralytics import YOLO
|
||||
|
||||
# 配置
|
||||
DATA_DIR = Path(__file__).parent / "datasets"
|
||||
MODEL_NAME = "yolov8n.pt" # 预训练模型
|
||||
OUTPUT_DIR = Path(__file__).parent / "runs"
|
||||
|
||||
def main():
|
||||
# 创建数据集配置文件(只包含野鸡类别)
|
||||
data_yaml = DATA_DIR / "data.yaml"
|
||||
print("创建数据集配置文件(只训练野鸡类别)...")
|
||||
with open(data_yaml, "w") as f:
|
||||
f.write(f"""# Observer 数据集配置 - 只识别野鸡
|
||||
path: {DATA_DIR}
|
||||
train: images/pheasant
|
||||
val: images/pheasant
|
||||
|
||||
# 类别
|
||||
nc: 1
|
||||
names: ['pheasant']
|
||||
""")
|
||||
|
||||
# 加载预训练模型
|
||||
print(f"加载预训练模型: {MODEL_NAME}")
|
||||
model = YOLO(MODEL_NAME)
|
||||
|
||||
# 训练模型
|
||||
print("开始训练...")
|
||||
results = model.train(
|
||||
data=str(data_yaml),
|
||||
epochs=50,
|
||||
imgsz=640,
|
||||
batch=16,
|
||||
name="observer_yolov8n",
|
||||
patience=20,
|
||||
save=True,
|
||||
plots=True
|
||||
)
|
||||
|
||||
print(f"\n训练完成!")
|
||||
print(f"最佳模型保存在: {OUTPUT_DIR / 'observer_yolov8n' / 'weights' / 'best.pt'}")
|
||||
|
||||
# 导出为 TFLite 格式
|
||||
print("\n导出为 TFLite 格式...")
|
||||
best_model_path = OUTPUT_DIR / "observer_yolov8n" / "weights" / "best.pt"
|
||||
if best_model_path.exists():
|
||||
best_model = YOLO(str(best_model_path))
|
||||
best_model.export(format="tflite", imgsz=320)
|
||||
print(f"TFLite 模型导出完成")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
使用 Hugging Face transformers 进行零样本目标检测
|
||||
使用 OWL-ViT 模型检测野鸡
|
||||
"""
|
||||
|
||||
import cv2
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from transformers import OwlViTProcessor, OwlViTForObjectDetection
|
||||
from PIL import Image
|
||||
|
||||
# 配置
|
||||
IMAGES_DIR = Path(__file__).parent / "datasets" / "images" / "pheasant"
|
||||
OUTPUT_DIR = Path(__file__).parent / "datasets" / "yolo_format"
|
||||
|
||||
# 检测配置
|
||||
CONFIDENCE_THRESHOLD = 0.1 # 置信度阈值
|
||||
|
||||
# 文本描述
|
||||
TEXT_PROMPTS = ["pheasant", "wild bird", "bird in grass"]
|
||||
|
||||
class ZeroShotDetector:
|
||||
def __init__(self):
|
||||
print("加载 OWL-ViT 模型...")
|
||||
self.processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32")
|
||||
self.model = OwlViTForObjectDetection.from_pretrained("google/owlvit-base-patch32")
|
||||
self.model.eval()
|
||||
print("OWL-ViT 模型加载完成")
|
||||
|
||||
def detect(self, image_path):
|
||||
"""检测图片中的野鸡"""
|
||||
# 读取图片
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
|
||||
# 准备输入
|
||||
inputs = self.processor(text=TEXT_PROMPTS, images=image, return_tensors="pt")
|
||||
|
||||
# 推理
|
||||
with torch.no_grad():
|
||||
outputs = self.model(**inputs)
|
||||
|
||||
# 获取结果
|
||||
target_sizes = torch.tensor([image.size[::-1]]) # [height, width]
|
||||
results = self.processor.post_process_grounded_object_detection(
|
||||
outputs, threshold=CONFIDENCE_THRESHOLD, target_sizes=target_sizes
|
||||
)[0]
|
||||
|
||||
# 解析结果
|
||||
detections = []
|
||||
for score, label, box in zip(results["scores"], results["labels"], results["boxes"]):
|
||||
box = box.tolist()
|
||||
detections.append({
|
||||
'bbox': box, # [x1, y1, x2, y2]
|
||||
'score': score.item(),
|
||||
'label': TEXT_PROMPTS[label],
|
||||
})
|
||||
|
||||
return detections
|
||||
|
||||
def convert_to_yolo_format(bbox, img_width, img_height):
|
||||
"""将边界框转换为 YOLO 格式 (cx, cy, w, h)"""
|
||||
x1, y1, x2, y2 = bbox
|
||||
cx = (x1 + x2) / 2 / img_width
|
||||
cy = (y1 + y2) / 2 / img_height
|
||||
w = (x2 - x1) / img_width
|
||||
h = (y2 - y1) / img_height
|
||||
return cx, cy, w, h
|
||||
|
||||
def main():
|
||||
# 初始化检测器
|
||||
detector = ZeroShotDetector()
|
||||
|
||||
# 创建输出目录
|
||||
(OUTPUT_DIR / "images" / "train").mkdir(parents=True, exist_ok=True)
|
||||
(OUTPUT_DIR / "labels" / "train").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 获取所有图片
|
||||
image_extensions = {".jpg", ".jpeg", ".png", ".bmp"}
|
||||
image_files = [
|
||||
f for f in IMAGES_DIR.iterdir()
|
||||
if f.suffix.lower() in image_extensions
|
||||
]
|
||||
|
||||
print(f"找到 {len(image_files)} 张图片")
|
||||
|
||||
# 处理每张图片
|
||||
labeled_count = 0
|
||||
for img_path in image_files:
|
||||
print(f"\n处理: {img_path.name}")
|
||||
|
||||
# 读取图片
|
||||
image = cv2.imread(str(img_path))
|
||||
if image is None:
|
||||
print(f" 跳过: 无法读取 {img_path.name}")
|
||||
continue
|
||||
|
||||
img_height, img_width = image.shape[:2]
|
||||
|
||||
# 检测野鸡
|
||||
detections = detector.detect(img_path)
|
||||
|
||||
# 生成标注
|
||||
labels = []
|
||||
for det in detections:
|
||||
bbox = det['bbox']
|
||||
score = det['score']
|
||||
|
||||
# 转换为 YOLO 格式
|
||||
cx, cy, w, h = convert_to_yolo_format(bbox, img_width, img_height)
|
||||
labels.append(f"0 {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}\n")
|
||||
|
||||
# 在图片上绘制检测框(用于可视化)
|
||||
x1, y1, x2, y2 = bbox
|
||||
cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
|
||||
cv2.putText(image, f"pheasant: {score:.2f}", (int(x1), int(y1) - 10),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
|
||||
|
||||
# 保存标注文件
|
||||
if len(labels) > 0:
|
||||
labeled_count += 1
|
||||
# 保存带标注的图片(用于可视化)
|
||||
cv2.imwrite(str(OUTPUT_DIR / "images" / "train" / img_path.name), image)
|
||||
|
||||
# 保存标注
|
||||
label_path = OUTPUT_DIR / "labels" / "train" / (img_path.stem + ".txt")
|
||||
with open(label_path, "w") as f:
|
||||
f.writelines(labels)
|
||||
|
||||
print(f" ✓ 标注了 {len(labels)} 个目标")
|
||||
else:
|
||||
print(f" - 未检测到野鸡")
|
||||
|
||||
print(f"\n完成!")
|
||||
print(f" 总图片数: {len(image_files)}")
|
||||
print(f" 有效标注: {labeled_count}")
|
||||
print(f" 输出目录: {OUTPUT_DIR}")
|
||||
|
||||
# 创建 data.yaml
|
||||
data_yaml = OUTPUT_DIR / "data.yaml"
|
||||
with open(data_yaml, "w") as f:
|
||||
f.write(f"""# Observer 数据集配置 - 只识别野鸡
|
||||
path: {OUTPUT_DIR}
|
||||
train: images/train
|
||||
val: images/train
|
||||
|
||||
# 类别
|
||||
nc: 1
|
||||
names: ['pheasant']
|
||||
""")
|
||||
print(f" 数据配置: {data_yaml}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user