feat：计算视频帧传来的差值

2026-04-02 15:29:49 +08:00
parent 1e828cc036
commit 51ea86e887
6 changed files with 294 additions and 26 deletions
--- a/backend/monitoring/services.py
+++ b/backend/monitoring/services.py
@@ -1,5 +1,6 @@
 from __future__ import annotations

+from collections import deque
 import json
 import math
 import os
@@ -20,6 +21,8 @@ GEOSTREAM_STALE_SECONDS = 15
 OMNISOCKET_CONFIG_PATH = PROJECT_ROOT / "config" / "omnisocket_demo.yaml"
 VIDEO_SOURCE_MODE = os.getenv("VIDEO_SOURCE_MODE", "auto").strip().lower()
 OMNISOCKET_FRAME_FRESH_SECONDS = 2.0
+VIDEO_TIMESTAMP_SAMPLE_SIZE = 10
+VIDEO_TIMESTAMP_MAX_SKEW_NS = 7 * 24 * 60 * 60 * 1_000_000_000


 def utc_iso_now() -> str:
@@ -38,6 +41,10 @@ class OmniSocketVideoReceiver:
        self._latest_frame: bytes | None = None
        self._latest_received_at = 0.0
        self._latest_sequence: int | None = None
+        self._latest_latency_ms: float | None = None
+        self._latest_timestamp_unit: str | None = None
+        self._latest_timestamp_endianness: str | None = None
+        self._latency_samples_ms: deque[float] = deque(maxlen=VIDEO_TIMESTAMP_SAMPLE_SIZE)
        self._frames_received = 0
        self._last_error = ""
        self._load_backend()
@@ -194,7 +201,7 @@ class OmniSocketVideoReceiver:
        )
        return session, int(video_cfg.get("buffer_bytes", 1024 * 1024))

-    def _extract_jpeg_frame(self, frame: bytes) -> bytes | None:
+    def _extract_jpeg_payload(self, frame: bytes) -> bytes | None:
        # 同时兼容两种帧格式：
        # 1. 纯 JPEG 二进制
        # 2. 前 8 字节是序号，后面才是真正的 JPEG 数据
@@ -204,11 +211,68 @@ class OmniSocketVideoReceiver:
            return frame[8:]
        return None

+    def _extract_jpeg_frame(self, frame: bytes) -> bytes | None:
+        jpeg_payload = self._extract_jpeg_payload(frame)
+        if jpeg_payload is None:
+            return None
+
+        eoi_index = jpeg_payload.rfind(b"\xff\xd9")
+        if eoi_index < 0:
+            return jpeg_payload
+
+        return jpeg_payload[: eoi_index + 2]
+
    def _extract_sequence(self, frame: bytes) -> int | None:
        if len(frame) >= 8 and not frame.startswith(b"\xff\xd8"):
            return int.from_bytes(frame[:8], "big")
        return None

+    def _extract_frame_tail(self, frame: bytes) -> bytes:
+        jpeg_payload = self._extract_jpeg_payload(frame)
+        if jpeg_payload is None:
+            return b""
+
+        eoi_index = jpeg_payload.rfind(b"\xff\xd9")
+        if eoi_index < 0:
+            return b""
+
+        trailer_start = eoi_index + 2
+        if trailer_start >= len(jpeg_payload):
+            return b""
+
+        return jpeg_payload[trailer_start:]
+
+    def _extract_frame_timestamp(self, frame: bytes) -> tuple[int, str, str] | None:
+        trailer = self._extract_frame_tail(frame)
+        if len(trailer) < 8:
+            return None
+
+        now_ns = time.time_ns()
+        raw_timestamp = trailer[-8:]
+        best_candidate: tuple[int, str, str] | None = None
+        best_distance_ns: int | None = None
+
+        for endianness in ("big", "little"):
+            value = int.from_bytes(raw_timestamp, endianness, signed=True)
+            if value <= 0:
+                continue
+
+            for unit, multiplier in (
+                ("ns", 1),
+                ("us", 1_000),
+                ("ms", 1_000_000),
+                ("s", 1_000_000_000),
+            ):
+                timestamp_ns = value * multiplier
+                distance_ns = abs(now_ns - timestamp_ns)
+                if distance_ns > VIDEO_TIMESTAMP_MAX_SKEW_NS:
+                    continue
+                if best_distance_ns is None or distance_ns < best_distance_ns:
+                    best_distance_ns = distance_ns
+                    best_candidate = (timestamp_ns, unit, endianness)
+
+        return best_candidate
+
    def _run(self) -> None:
        # 后台持续接收循环：
        # connect -> recv_into(buffer) -> 按 body_len 截出有效内容 -> 把最新 JPEG 帧缓存在内存里
@@ -234,11 +298,25 @@ class OmniSocketVideoReceiver:
                        self._last_error = "received non-JPEG binary frame"
                        continue

+                    timestamp_meta = self._extract_frame_timestamp(frame)
+                    latency_ms = None
+                    if timestamp_meta is not None:
+                        timestamp_ns, unit, endianness = timestamp_meta
+                        latency_ms = round((time.time_ns() - timestamp_ns) / 1_000_000, 3)
+                    else:
+                        unit = None
+                        endianness = None
+
                    with self._lock:
                        # 缓存：这里只保留最新的一张 JPEG 帧，供 Web 接口直接返回给前端。
                        self._latest_frame = jpeg_frame
                        self._latest_received_at = time.time()
                        self._latest_sequence = self._extract_sequence(frame)
+                        self._latest_latency_ms = latency_ms
+                        self._latest_timestamp_unit = unit
+                        self._latest_timestamp_endianness = endianness
+                        if latency_ms is not None:
+                            self._latency_samples_ms.append(latency_ms)
                        self._frames_received += 1
            except Exception as error:  # pragma: no cover - 运行时集成路径
                self._last_error = str(error)
@@ -271,6 +349,26 @@ class OmniSocketVideoReceiver:
            has_recent_frame = self._latest_frame is not None and (
                time.time() - self._latest_received_at <= OMNISOCKET_FRAME_FRESH_SECONDS
            )
+            if has_recent_frame and self._latest_latency_ms is not None:
+                timing_status = {
+                    "available": True,
+                    "latest_delta_ms": self._latest_latency_ms,
+                    "delta_samples_ms": list(reversed(self._latency_samples_ms)),
+                    "sample_count": len(self._latency_samples_ms),
+                    "sample_window_size": VIDEO_TIMESTAMP_SAMPLE_SIZE,
+                    "timestamp_unit": self._latest_timestamp_unit,
+                    "timestamp_endianness": self._latest_timestamp_endianness,
+                }
+            else:
+                timing_status = {
+                    "available": False,
+                    "latest_delta_ms": None,
+                    "delta_samples_ms": [],
+                    "sample_count": 0,
+                    "sample_window_size": VIDEO_TIMESTAMP_SAMPLE_SIZE,
+                    "timestamp_unit": None,
+                    "timestamp_endianness": None,
+                }
            return {
                "backend_ready": self._session_cls is not None,
                "mode": VIDEO_SOURCE_MODE,
@@ -284,6 +382,7 @@ class OmniSocketVideoReceiver:
                "relay_via": str(transport_cfg.get("relay_via", "")),
                "peer_id": str(video_cfg.get("peer_id", "")),
                "buffer_bytes": int(video_cfg.get("buffer_bytes", 0)),
+                "timing": timing_status,
            }


@@ -305,6 +404,7 @@ class VideoFrameService:
                "frame_dir": str(JPEG_FRAME_DIR),
                "source_detail": f"peer stream active, frames={receiver_status['frames_received']}",
                "receiver": receiver_status,
+                "timing": receiver_status["timing"],
            }

        wait_detail = receiver_status["last_error"] or (
@@ -318,6 +418,7 @@ class VideoFrameService:
            "frame_dir": str(JPEG_FRAME_DIR),
            "source_detail": wait_detail,
            "receiver": receiver_status,
+            "timing": receiver_status["timing"],
        }

    def get_next_frame(self) -> bytes:
--- a/config/omnisocket_demo.yaml
+++ b/config/omnisocket_demo.yaml
@@ -1,6 +1,6 @@
 transport:
-  server_addr: "127.0.0.1:10909"
-  relay_via: ""
+  server_addr: ""
+  relay_via: "106.55.173.235:10909"
  bind_ip: ""
  bind_device: ""

--- a/frontend/src/App.vue
+++ b/frontend/src/App.vue
@@ -63,24 +63,26 @@ const navItems = [
 .app-shell {
  width: min(1440px, calc(100% - 32px));
  margin: 0 auto;
-  padding: 22px 0 40px;
+  padding: 0 0 40px;
 }

 .topbar {
  position: sticky;
-  top: 16px;
-  z-index: 20;
+  top: 0;
+  z-index: 100;
  display: flex;
  justify-content: space-between;
  align-items: center;
  gap: 20px;
  padding: 14px 18px;
  margin-bottom: 24px;
-  border-radius: 24px;
-  background: rgba(8, 14, 26, 0.82);
-  border: 1px solid rgba(133, 147, 169, 0.2);
-  box-shadow: 0 18px 40px rgba(0, 0, 0, 0.18);
-  backdrop-filter: blur(16px);
+  border-radius: 0 0 24px 24px;
+  background: linear-gradient(180deg, #0a1324 0%, #08101d 100%);
+  border: 1px solid rgba(133, 147, 169, 0.22);
+  border-top: none;
+  box-shadow: 0 18px 40px rgba(0, 0, 0, 0.28);
+  overflow: hidden;
+  isolation: isolate;
 }

 :global(.panel) {
@@ -164,7 +166,6 @@ const navItems = [

@media (max-width: 960px) {
  .topbar {
-    position: static;
    flex-direction: column;
    align-items: stretch;
  }
--- a/frontend/src/components/VideoPanel.vue
+++ b/frontend/src/components/VideoPanel.vue
@@ -1,37 +1,82 @@
 <script setup lang="ts">
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'

-import { buildVideoFrameUrl } from '@/lib/api'
+import { buildVideoFrameUrl, fetchVideoStatus } from '@/lib/api'
 import type { VideoStatus } from '@/types'

 const props = defineProps<{
  video: VideoStatus | null
 }>()

+const STATUS_REFRESH_MS = 300
+
+const liveVideo = ref<VideoStatus | null>(props.video)
 const frameUrl = ref(buildVideoFrameUrl(0))
-const currentFps = computed(() => props.video?.fps ?? 30)
-const canRequestFrames = computed(() => props.video?.available === true)
+const displayVideo = computed(() => liveVideo.value ?? props.video)
+const currentFps = computed(() => displayVideo.value?.fps ?? 30)
+const canRequestFrames = computed(() => displayVideo.value?.available === true)
 const modeLabel = computed(() => {
-  if (!props.video) {
+  if (!displayVideo.value) {
    return '正在获取视频状态'
  }
-  if (props.video.source_mode === 'omnisocket-jpeg-live') {
-    return `${props.video.fps} FPS 实时接收`
+  if (displayVideo.value.source_mode === 'omnisocket-jpeg-live') {
+    return `${displayVideo.value.fps} FPS 实时接收`
  }
-  if (props.video.source_mode === 'omnisocket-waiting') {
+  if (displayVideo.value.source_mode === 'omnisocket-waiting') {
    return '未实时获取真实值'
  }
-  return `${props.video.fps} FPS`
+  return `${displayVideo.value.fps} FPS`
 })
 const placeholderText = computed(() => {
-  if (!props.video) {
+  if (!displayVideo.value) {
    return '正在获取视频状态...'
  }
  return '未实时获取真实值'
 })
+const latencyLabels = computed(() => {
+  const sampleWindowSize = displayVideo.value?.timing?.sample_window_size ?? 10
+  const samples = displayVideo.value?.timing?.delta_samples_ms ?? []
+  return Array.from({ length: sampleWindowSize }, (_, index) => samples[index] ?? null)
+})
+const timingHeadline = computed(() => {
+  const latest = displayVideo.value?.timing?.latest_delta_ms
+  if (latest == null) {
+    return '等待帧尾时间'
+  }
+  return `最新 ${latest.toFixed(1)} ms`
+})
+const timingHint = computed(() => {
+  const timing = displayVideo.value?.timing
+  if (!timing?.available) {
+    return '当前还没有从 JPEG 结尾后的尾字节里解析到时间戳，标签会在收到有效帧尾时间后自动填充。'
+  }
+
+  const unitText = timing.timestamp_unit ? `，单位按 ${timing.timestamp_unit}` : ''
+  const endiannessText = timing.timestamp_endianness
+    ? `，字节序按 ${timing.timestamp_endianness}`
+    : ''
+  return `最近 ${timing.sample_window_size} 个差值样本，面板按 ${STATUS_REFRESH_MS} ms 刷新一组${unitText}${endiannessText}。`
+})

 let frameTimer: number | null = null
+let statusTimer: number | null = null
 let frameKey = 0
+let statusRequestPending = false
+
+async function refreshStatus() {
+  if (statusRequestPending) {
+    return
+  }
+
+  statusRequestPending = true
+  try {
+    liveVideo.value = await fetchVideoStatus()
+  } catch {
+    // 保持当前已显示状态，避免短暂请求失败把面板内容清空。
+  } finally {
+    statusRequestPending = false
+  }
+}

 function refreshFrame() {
  if (!canRequestFrames.value) {
@@ -58,7 +103,20 @@ function startFrameLoop() {
  }, intervalMs)
 }

+function startStatusLoop() {
+  if (statusTimer != null) {
+    window.clearInterval(statusTimer)
+    statusTimer = null
+  }
+
+  void refreshStatus()
+  statusTimer = window.setInterval(() => {
+    void refreshStatus()
+  }, STATUS_REFRESH_MS)
+}
+
 onMounted(() => {
+  startStatusLoop()
  startFrameLoop()
 })

@@ -66,8 +124,19 @@ onUnmounted(() => {
  if (frameTimer != null) {
    window.clearInterval(frameTimer)
  }
+  if (statusTimer != null) {
+    window.clearInterval(statusTimer)
+  }
 })

+watch(
+  () => props.video,
+  (nextVideo) => {
+    liveVideo.value = nextVideo
+  },
+  { immediate: true },
+)
+
 watch([currentFps, canRequestFrames], () => {
  startFrameLoop()
 })
@@ -80,8 +149,8 @@ watch([currentFps, canRequestFrames], () => {
        <p class="eyebrow">Video</p>
        <h2>JPEG 视频流</h2>
      </div>
-      <span class="badge" :class="{ bad: !video?.available }">
-        {{ video?.source_mode ?? 'loading' }}
+      <span class="badge" :class="{ bad: !displayVideo?.available }">
+        {{ displayVideo?.source_mode ?? 'loading' }}
      </span>
    </div>

@@ -100,7 +169,7 @@ watch([currentFps, canRequestFrames], () => {
    <div class="stats">
      <div class="stat-card">
        <span>帧源</span>
-        <strong>{{ video?.frame_count ?? '--' }} 张 JPEG</strong>
+        <strong>{{ displayVideo?.frame_count ?? '--' }} 张 JPEG</strong>
      </div>
      <div class="stat-card">
        <span>当前模式</span>
@@ -108,13 +177,33 @@ watch([currentFps, canRequestFrames], () => {
      </div>
    </div>

+    <div class="timing-panel">
+      <div class="timing-head">
+        <span>帧尾时间差</span>
+        <strong>{{ timingHeadline }}</strong>
+      </div>
+      <div class="timing-grid">
+        <span
+          v-for="(sample, index) in latencyLabels"
+          :key="index"
+          class="timing-label"
+          :class="{ empty: sample == null }"
+        >
+          {{ sample == null ? '--' : `${sample.toFixed(1)} ms` }}
+        </span>
+      </div>
+      <p class="hint subtle">
+        {{ timingHint }}
+      </p>
+    </div>
+
    <p class="hint">
      这里只有在后端已经收到 OmniSocket 的真实 JPEG 帧时，才会开始逐帧请求并显示画面。
      如果当前没有真实帧，页面会保持占位提示，不再回退测试视频流。
    </p>

    <p class="hint subtle">
-      当前帧源状态：{{ video?.source_detail ?? '暂无' }}
+      当前帧源状态：{{ displayVideo?.source_detail ?? '暂无' }}
    </p>
  </section>
 </template>
@@ -213,6 +302,57 @@ h2 {
  font-size: 18px;
 }

+.timing-panel {
+  display: grid;
+  gap: 12px;
+  padding: 14px;
+  border-radius: 18px;
+  background: rgba(7, 14, 26, 0.88);
+  border: 1px solid rgba(133, 147, 169, 0.18);
+}
+
+.timing-head {
+  display: flex;
+  justify-content: space-between;
+  gap: 12px;
+  align-items: center;
+  color: #cfd7e6;
+}
+
+.timing-head span {
+  color: #8d99b3;
+  font-size: 12px;
+}
+
+.timing-head strong {
+  font-size: 16px;
+}
+
+.timing-grid {
+  display: grid;
+  grid-template-columns: repeat(5, minmax(0, 1fr));
+  gap: 8px;
+}
+
+.timing-label {
+  display: grid;
+  place-items: center;
+  min-height: 40px;
+  padding: 8px 10px;
+  border-radius: 12px;
+  background: rgba(91, 122, 255, 0.12);
+  border: 1px solid rgba(91, 122, 255, 0.28);
+  color: #dce4ff;
+  font-size: 12px;
+  font-weight: 700;
+}
+
+.timing-label.empty {
+  background: rgba(133, 147, 169, 0.08);
+  border-color: rgba(133, 147, 169, 0.18);
+  color: #7e8aa5;
+}
+
 .hint {
  margin: 0;
  color: #8d99b3;
@@ -227,5 +367,9 @@ h2 {
  .stats {
    grid-template-columns: 1fr;
  }
+
+  .timing-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
 }
 </style>
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -1,4 +1,4 @@
-import type { DashboardSnapshot } from '@/types'
+import type { DashboardSnapshot, VideoStatus } from '@/types'

 const envBaseUrl = import.meta.env.VITE_API_BASE_URL as string | undefined

@@ -16,6 +16,10 @@ export function fetchDashboardSnapshot() {
  return fetchJson<DashboardSnapshot>('/api/dashboard/')
 }

+export function fetchVideoStatus() {
+  return fetchJson<VideoStatus>('/api/video/status/')
+}
+
 export function buildVideoFrameUrl(frameKey: number) {
  return `${API_BASE}/api/video/frame/?frame=${frameKey}&t=${Date.now()}`
 }
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -32,6 +32,15 @@ export interface VideoStatus {
  fps: number
  frame_dir: string
  source_detail?: string
+  timing?: {
+    available: boolean
+    latest_delta_ms: number | null
+    delta_samples_ms: number[]
+    sample_count: number
+    sample_window_size: number
+    timestamp_unit: string | null
+    timestamp_endianness: string | null
+  }
  receiver?: {
    backend_ready: boolean
    mode: string
@@ -45,6 +54,15 @@ export interface VideoStatus {
    relay_via?: string
    peer_id?: string
    buffer_bytes?: number
+    timing?: {
+      available: boolean
+      latest_delta_ms: number | null
+      delta_samples_ms: number[]
+      sample_count: number
+      sample_window_size: number
+      timestamp_unit: string | null
+      timestamp_endianness: string | null
+    }
  }
 }