meiqi/TG3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: keep RGB-D captures only on workstation

Browse Source
This commit is contained in:
meiqi
2026-08-10 17:14:44 +08:00
parent d02b99d2aa
commit 67c9b65768
15 changed files with 2817 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tg3_local_teleop/README.md
View File

@@ -167,21 +167,19 @@ OmniSocket 进程；在此之前桥会保持最后一帧目标。未 START 或
为避免 1280x720 原始 RGB+深度每台相机约 `8.3 GB/min` 的持续写盘负载，默认录制
`color/image_raw/compressed`（JPEG）和 `depth/image_raw/compressedDepth`（16UC1
无损 PNG）。现场单台腰部相机实测合计约 `22–25 MB/s`（约 `1.3–1.5 GB/min`，随
场景变化）。腰部图像、内参、Metadata 和 `/tf_static` 是 required topics；两路图像还
必须在整条 bag 内分别达到至少 `20 Hz` 平均频率。相机服务未发布或图像严重掉流时，
L3 启动或最终校验会失败，不会生成缺图像却标记 READY 的 episode。头部话题在白名单
中但不是 required：当前
`orbbec_head.service` 未启动不影响腰部数采；将来在 episode 开始前启动后，rosbag 会
自动发现并采集，无需再改配置。静态 TF 会被保存用于相机外参，但当前收尾只验证该
Topic 非空，不解析每条 TF 的 frame 语义。
场景变化）。头、腰两组相机都不是 required topics：服务在运行就由 rosbag 自动发现并
采集，未运行不阻塞 L3 或核心 READY。manifest 对每组记录 `absent`、`healthy`、
`partial` 或 `low_rate`，并保存各话题消息数和图像平均频率；相机异常会产生明确质量
警告，但不会丢弃其余机器人控制数据。静态 TF 也尽力采集，不作为启动门槛。
机器人暂存目录：
```text
/home/nvidia/tg3_data_collection/
active/ # 尚未完成，不能取走
ready/ # 已正常收尾、校验并生成 READY
failed/ # 启动、磁盘、进程或校验失败，保留供诊断
active/ # 录制期间的临时数据，不能取走
ready/ # 等待 PS 本机完成持久化验真；随后自动回收
failed/ # 当前策略不保留失败 payload
deleting/ # 固定根删除助手的短暂原子隔离目录
```
MCAP 使用 `zstd_fast`、64 MiB cache、每 300 秒分片；每条 episode 最长 30 分钟，
@@ -191,6 +189,12 @@ MCAP 使用 `zstd_fast`、64 MiB cache、每 300 秒分片；每条 episode 最
`manifest.json` 和 `READY`，最后才原子进入 `ready/`。Nvidia 需已安装 `python3-yaml`
（现场已验证 PyYAML 6.0.1）。
当前 `retain_failed_episodes=false`：失败原因仍进入 recorder 状态和 journal，但失败
MCAP 不留在机器人磁盘。成功 episode 必须先在 PS 本机完成逐文件 SHA-256、fsync、
原子发布并生成 `VERIFIED`，然后由固定根目录删除助手按 episode ID + manifest SHA
精确回收；它不能访问 `active/`、`failed/` 或数据根目录。PS 断线时 `ready/` 会临时保留，
避免尚未落到本机的数据丢失，恢复后自动续传和回收。
相机由厂家系统服务管理，本项目只订阅、不自动启停。录制前检查或按需启动：
```bash
@@ -224,7 +228,8 @@ find /home/nvidia/tg3_data_collection/ready -mindepth 1 -maxdepth 1 -type d
```
完成 episode 由 PS 本机服务校验后保存到项目 `Data_Get`，详见
`../tg3_data_collection/README.md`。机器人 `ready/` 中的副本不会自动删除。
`../tg3_data_collection/README.md`。本机 `VERIFIED` 持久化后，机器人对应 `ready/`
副本会自动删除；EAI 不保存数采 payload。
## 完整重启顺序

48
tg3_local_teleop/config.toml
View File

@@ -75,20 +75,17 @@ control_topic = "/tg3/data_collection/control"
status_topic = "/tg3/data_collection/status"
iarm_frame_topic = "/tg3/data_collection/iarm_frame"
base_directory = "/home/nvidia/tg3_data_collection"
# Failed/incomplete payloads are discarded after the error is reported through
# recorder status and journal. Completed READY episodes remain only until the
# PS sync has durably verified its local copy and requests exact-episode removal.
retain_failed_episodes = false
minimum_free_gib = 100.0
max_duration_seconds = 1800.0
# A camera endpoint merely existing is insufficient: completed episodes must
# also sustain this conservative average rate across the rosbag data span.
# The driver targets 30 Hz; 20 Hz tolerates startup/discovery jitter while
# rejecting a stream that only delivered a few frames or stopped mid-episode.
minimum_topic_rates_hz."/ob_camera_waist/color/image_raw/compressed" = 20.0
minimum_topic_rates_hz."/ob_camera_waist/depth/image_raw/compressedDepth" = 20.0
# Explicit data whitelist; never replace this with `-a` on the live robot.
# RGB is recorded through image_transport JPEG and 16UC1 depth through its
# lossless compressedDepth PNG transport. Waist RGB-D is required because that
# camera is enabled on this robot. Head RGB-D stays optional: rosbag discovers
# and records it automatically whenever orbbec_head.service is running.
# lossless compressedDepth PNG transport. Both RGB-D cameras are optional:
# rosbag discovers and records either one whenever its service is running.
# /robot_state is the authoritative measured robot state.
topics = [
"/robot_state",
@@ -137,18 +134,43 @@ required_topics = [
"/left_hand/motor_status",
"/right_hand/motor_status",
"/hric/robot/rl_state",
"/tg3/data_collection/control",
"/tg3/data_collection/status",
"/tg3/data_collection/iarm_frame",
]
# Optional sensor groups never block capture or core READY. Their message
# counts and whole-episode average rates are recorded in manifest.json as
# absent/healthy/partial/low_rate quality observations. The Orbbec driver
# targets 30 Hz; 20 Hz leaves room for discovery and compression jitter.
[data_collection.optional_topic_groups.head_rgbd]
topics = [
"/ob_camera_head/color/image_raw/compressed",
"/ob_camera_head/color/camera_info",
"/ob_camera_head/color/metadata",
"/ob_camera_head/depth/image_raw/compressedDepth",
"/ob_camera_head/depth/camera_info",
"/ob_camera_head/depth/metadata",
]
[data_collection.optional_topic_groups.head_rgbd.minimum_topic_rates_hz]
"/ob_camera_head/color/image_raw/compressed" = 20.0
"/ob_camera_head/depth/image_raw/compressedDepth" = 20.0
[data_collection.optional_topic_groups.waist_rgbd]
topics = [
"/ob_camera_waist/color/image_raw/compressed",
"/ob_camera_waist/color/camera_info",
"/ob_camera_waist/color/metadata",
"/ob_camera_waist/depth/image_raw/compressedDepth",
"/ob_camera_waist/depth/camera_info",
"/ob_camera_waist/depth/metadata",
"/tf_static",
"/tg3/data_collection/control",
"/tg3/data_collection/status",
"/tg3/data_collection/iarm_frame",
]
[data_collection.optional_topic_groups.waist_rgbd.minimum_topic_rates_hz]
"/ob_camera_waist/color/image_raw/compressed" = 20.0
"/ob_camera_waist/depth/image_raw/compressedDepth" = 20.0
[locomotion]
# The two original immediate bindings are independent: right C + left-stick
# vertical controls translation; left Z + right-stick horizontal controls

252
tg3_local_teleop/data_collection.py
View File

@@ -32,6 +32,15 @@ import yaml
ToggleAction = Literal["start", "stop"]
def _valid_ros_topic(topic: Any) -> bool:
return (
isinstance(topic, str)
and topic.startswith("/")
and topic.strip() == topic
and not any(character.isspace() for character in topic)
)
def left_joystick_pressed(data: Mapping[str, Any]) -> bool | None:
"""Strictly parse the live-frame ``button_joystick.left`` value.
@@ -195,6 +204,55 @@ class RecordingToggleGate:
return "active" if self.active else "idle"
@dataclass(frozen=True)
class OptionalTopicGroupConfig:
"""Non-fatal completeness and rate checks for an optional sensor group."""
topics: Sequence[str]
minimum_topic_rates_hz: Mapping[str, float] = field(default_factory=dict)
def __post_init__(self) -> None:
topics = (
tuple(self.topics)
if isinstance(self.topics, Sequence)
and not isinstance(self.topics, str)
else ()
)
if not topics:
raise ValueError("optional topic group must contain at least one topic")
if any(not _valid_ros_topic(topic) for topic in topics):
raise ValueError(
"optional topic group topics must be absolute ROS topic names"
)
if len(set(topics)) != len(topics):
raise ValueError("optional topic group topics must not contain duplicates")
if not isinstance(self.minimum_topic_rates_hz, Mapping):
raise ValueError("optional minimum topic rates must be a mapping")
rates: dict[str, float] = {}
for topic, rate in self.minimum_topic_rates_hz.items():
if topic not in topics:
raise ValueError(
"optional minimum-rate topics must be a subset of group topics"
)
if (
isinstance(rate, bool)
or not isinstance(rate, (int, float))
or not math.isfinite(rate)
or rate <= 0.0
):
raise ValueError(
f"optional minimum topic rate for {topic!r} must be "
"positive and finite"
)
rates[topic] = float(rate)
object.__setattr__(self, "topics", topics)
object.__setattr__(
self,
"minimum_topic_rates_hz",
MappingProxyType(rates),
)
@dataclass(frozen=True)
class RecorderConfig:
"""Static configuration for :class:`DataRecorderManager`."""
@@ -203,6 +261,10 @@ class RecorderConfig:
topics: Sequence[str] = ()
required_topics: Sequence[str] = ()
minimum_topic_rates_hz: Mapping[str, float] = field(default_factory=dict)
optional_topic_groups: Mapping[str, OptionalTopicGroupConfig] = field(
default_factory=dict
)
retain_failed_episodes: bool = True
minimum_free_bytes: int = 5 * 1024**3
max_duration_seconds: float = 30 * 60.0
poll_interval_seconds: float = 0.1
@@ -219,25 +281,13 @@ class RecorderConfig:
minimum_topic_rates: dict[str, float] = {}
if not topics:
raise ValueError("at least one recording topic is required")
if any(
not isinstance(topic, str)
or not topic.startswith("/")
or topic.strip() != topic
or any(character.isspace() for character in topic)
for topic in topics
):
if any(not _valid_ros_topic(topic) for topic in topics):
raise ValueError("recording topics must be absolute ROS topic names")
if len(set(topics)) != len(topics):
raise ValueError("recording topics must not contain duplicates")
if len(set(required_topics)) != len(required_topics):
raise ValueError("required topics must not contain duplicates")
if any(
not isinstance(topic, str)
or not topic.startswith("/")
or topic.strip() != topic
or any(character.isspace() for character in topic)
for topic in required_topics
):
if any(not _valid_ros_topic(topic) for topic in required_topics):
raise ValueError("required topics must be absolute ROS topic names")
if not set(required_topics).issubset(topics):
raise ValueError("required topics must be a subset of recording topics")
@@ -258,6 +308,45 @@ class RecorderConfig:
f"minimum topic rate for {topic!r} must be positive and finite"
)
minimum_topic_rates[topic] = float(rate)
if not isinstance(self.optional_topic_groups, Mapping):
raise ValueError("optional topic groups must be a mapping")
optional_groups: dict[str, OptionalTopicGroupConfig] = {}
grouped_topics: set[str] = set()
for name, group in self.optional_topic_groups.items():
if (
not isinstance(name, str)
or not name
or len(name) > 64
or any(
character
not in "-_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
for character in name
)
):
raise ValueError(
"optional topic group names must contain 1-64 safe characters"
)
if not isinstance(group, OptionalTopicGroupConfig):
raise ValueError(
f"optional topic group {name!r} has an invalid configuration"
)
group_topics = set(group.topics)
if not group_topics.issubset(topics):
raise ValueError(
f"optional topic group {name!r} must be a subset of recording topics"
)
if group_topics.intersection(required_topics):
raise ValueError(
f"optional topic group {name!r} must be disjoint from required topics"
)
overlap = group_topics.intersection(grouped_topics)
if overlap:
raise ValueError(
"optional topic groups must be disjoint; repeated topics: "
+ ", ".join(sorted(overlap))
)
grouped_topics.update(group_topics)
optional_groups[name] = group
if (
isinstance(self.minimum_free_bytes, bool)
or not isinstance(self.minimum_free_bytes, int)
@@ -277,6 +366,8 @@ class RecorderConfig:
raise ValueError("ROS 2 executable must be a non-empty string")
if type(self.validate_bag_info) is not bool:
raise ValueError("validate bag info must be a bool")
if type(self.retain_failed_episodes) is not bool:
raise ValueError("retain failed episodes must be a bool")
object.__setattr__(self, "base_directory", base)
object.__setattr__(self, "topics", topics)
object.__setattr__(self, "required_topics", required_topics)
@@ -285,6 +376,11 @@ class RecorderConfig:
"minimum_topic_rates_hz",
MappingProxyType(minimum_topic_rates),
)
object.__setattr__(
self,
"optional_topic_groups",
MappingProxyType(optional_groups),
)
class RecorderProcess(Protocol):
@@ -325,6 +421,8 @@ class _Episode:
required_topic_message_counts: dict[str, int] | None = None
minimum_topic_message_counts: dict[str, int] | None = None
observed_topic_rates_hz: dict[str, float] | None = None
optional_topic_groups: dict[str, Any] | None = None
data_quality_warnings: list[str] | None = None
metadata_duration_nanoseconds: int | None = None
process: RecorderProcess | None = None
stdout_stream: Any = None
@@ -767,6 +865,8 @@ class DataRecorderManager:
episode.minimum_topic_message_counts
),
"observed_topic_rates_hz": episode.observed_topic_rates_hz,
"optional_topic_groups": episode.optional_topic_groups,
"data_quality_warnings": episode.data_quality_warnings,
"metadata_duration_nanoseconds": (
episode.metadata_duration_nanoseconds
),
@@ -859,6 +959,7 @@ class DataRecorderManager:
topic: all_counts.get(topic, 0) for topic in self.config.topics
}
episode.required_topic_message_counts = required_counts
self._observe_optional_topic_groups(episode, information, all_counts)
missing = [
topic for topic in self.config.required_topics if topic not in all_counts
]
@@ -877,6 +978,101 @@ class DataRecorderManager:
self._validate_minimum_topic_rates(episode, information, all_counts)
return required_counts
def _observe_optional_topic_groups(
self,
episode: _Episode,
information: Mapping[str, Any],
all_counts: Mapping[str, int],
) -> None:
"""Classify optional sensor quality without invalidating core data."""
duration_nanoseconds: int | None = None
duration = information.get("duration")
if isinstance(duration, Mapping):
candidate = duration.get("nanoseconds")
if (
type(candidate) is int
and candidate > 0
):
duration_nanoseconds = candidate
episode.metadata_duration_nanoseconds = candidate
duration_seconds = (
None
if duration_nanoseconds is None
else duration_nanoseconds / 1_000_000_000.0
)
observations: dict[str, Any] = {}
warnings: list[str] = []
for name, group in self.config.optional_topic_groups.items():
counts = {
topic: int(all_counts.get(topic, 0)) for topic in group.topics
}
empty_topics = [
topic for topic, count in counts.items() if count <= 0
]
all_absent = len(empty_topics) == len(group.topics)
minimum_counts: dict[str, int | None] = {}
observed_rates: dict[str, float | None] = {}
below_rate: list[str] = []
for topic, minimum_rate in group.minimum_topic_rates_hz.items():
if duration_seconds is None:
minimum_counts[topic] = None
observed_rates[topic] = None
if not all_absent:
below_rate.append(topic)
continue
minimum_count = math.floor(duration_seconds * minimum_rate)
observed_rate = counts[topic] / duration_seconds
minimum_counts[topic] = minimum_count
observed_rates[topic] = observed_rate
if not all_absent and counts[topic] < minimum_count:
below_rate.append(topic)
if all_absent:
state = "absent"
elif empty_topics:
state = "partial"
warnings.append(
f"optional topic group {name!r} is partial; zero-message "
"topics: " + ", ".join(empty_topics)
)
elif below_rate:
state = "low_rate"
if duration_seconds is None:
warnings.append(
f"optional topic group {name!r} rate could not be "
"validated because bag duration is unavailable"
)
else:
details = [
f"{topic}={observed_rates[topic]:.3f}Hz<"
f"{group.minimum_topic_rates_hz[topic]:.3f}Hz"
for topic in below_rate
]
warnings.append(
f"optional topic group {name!r} is below its observed "
"minimum rate: " + "; ".join(details)
)
else:
state = "healthy"
observations[name] = {
"state": state,
"topics": list(group.topics),
"topic_message_counts": counts,
"minimum_topic_rates_hz": dict(
group.minimum_topic_rates_hz
),
"minimum_topic_message_counts": minimum_counts,
"observed_topic_rates_hz": observed_rates,
"zero_message_topics": empty_topics,
"below_minimum_rate_topics": below_rate,
}
episode.optional_topic_groups = observations
episode.data_quality_warnings = warnings
def _validate_minimum_topic_rates(
self,
episode: _Episode,
@@ -1036,6 +1232,31 @@ class DataRecorderManager:
) -> tuple[Path | None, str]:
if not episode.active_directory.exists():
return None, "active episode directory is missing"
if not self.config.retain_failed_episodes:
try:
active_root = Path(self.config.base_directory) / "active"
target = episode.active_directory
if (
target.parent != active_root
or target.name != episode.request.episode_id
or target.is_symlink()
or active_root.is_symlink()
):
raise RecordingError(
"refusing to discard a failed episode outside its "
"fixed active root"
)
shutil.rmtree(target)
try:
self._sync_directory(active_root)
except OSError:
pass
return None, ""
except Exception as discard_error:
return None, (
"failed to discard project-owned failed episode: "
f"{type(discard_error).__name__}: {discard_error}"
)
try:
ready_marker = episode.active_directory / "READY"
if ready_marker.exists():
@@ -1063,6 +1284,8 @@ class DataRecorderManager:
episode.minimum_topic_message_counts
),
"observed_topic_rates_hz": episode.observed_topic_rates_hz,
"optional_topic_groups": episode.optional_topic_groups,
"data_quality_warnings": episode.data_quality_warnings,
"metadata_duration_nanoseconds": (
episode.metadata_duration_nanoseconds
),
@@ -1242,6 +1465,7 @@ class DataRecorderManager:
__all__ = [
"DataRecorderManager",
"OptionalTopicGroupConfig",
"RecorderConfig",
"RecordingError",
"RecordingToggleGate",

49
tg3_local_teleop/data_recorder_node.py
View File

@@ -29,7 +29,11 @@ from rclpy.qos import (
from rclpy.utilities import remove_ros_args
from std_msgs.msg import String
from data_collection import DataRecorderManager, RecorderConfig
from data_collection import (
DataRecorderManager,
OptionalTopicGroupConfig,
RecorderConfig,
)
from data_recorder_protocol import (
RecorderControlProtocol,
topics_without_publishers,
@@ -102,6 +106,41 @@ def _minimum_topic_rates(value: Any) -> dict[str, float]:
return result
def _optional_topic_groups(
value: Any,
) -> dict[str, OptionalTopicGroupConfig]:
name = "optional_topic_groups"
if not isinstance(value, dict):
raise ValueError(f"data_collection.{name} must be a TOML table")
result: dict[str, OptionalTopicGroupConfig] = {}
for group_name, raw_group in value.items():
if not isinstance(raw_group, dict):
raise ValueError(
f"data_collection.{name}.{group_name} must be a TOML table"
)
unknown = sorted(
set(raw_group) - {"topics", "minimum_topic_rates_hz"}
)
if unknown:
raise ValueError(
f"data_collection.{name}.{group_name} has unknown keys: "
+ ", ".join(unknown)
)
topics = _topic_list(
raw_group.get("topics", []),
f"{name}.{group_name}.topics",
allow_empty=False,
)
rates = _minimum_topic_rates(
raw_group.get("minimum_topic_rates_hz", {})
)
result[group_name] = OptionalTopicGroupConfig(
topics=topics,
minimum_topic_rates_hz=rates,
)
return result
@dataclass(frozen=True)
class SupervisorSettings:
enabled: bool
@@ -150,6 +189,9 @@ def load_settings(path: Path | str) -> SupervisorSettings:
minimum_topic_rates = _minimum_topic_rates(
section.get("minimum_topic_rates_hz", {})
)
optional_topic_groups = _optional_topic_groups(
section.get("optional_topic_groups", {})
)
unknown_rate_topics = sorted(
set(minimum_topic_rates) - set(required_topics)
)
@@ -187,6 +229,11 @@ def load_settings(path: Path | str) -> SupervisorSettings:
topics=topics,
required_topics=required_topics,
minimum_topic_rates_hz=minimum_topic_rates,
optional_topic_groups=optional_topic_groups,
retain_failed_episodes=_boolean(
section.get("retain_failed_episodes", True),
"retain_failed_episodes",
),
minimum_free_bytes=int(minimum_free_gib * 1024**3),
max_duration_seconds=max_duration,
poll_interval_seconds=0.1,

537
tg3_local_teleop/delete_ready_episode.py Executable file
View File

@@ -0,0 +1,537 @@
#!/usr/bin/env python3
"""Safely and idempotently remove one verified recorder ``ready`` episode.
This helper intentionally has no command-line option for either root directory.
It can only move a direct child of the fixed recorder ``ready`` directory into
the fixed ``deleting`` staging directory and remove that tombstone. The caller
must provide both the episode id and the SHA-256 of the exact manifest that was
verified and durably published on the PS workstation.
Before recursive removal, an authorization ledger is fsynced in the fixed
``delete_ledger`` sibling directory. It survives partial tombstone deletion and
is removed only after the tree cleanup itself is durable.
"""
from __future__ import annotations
import argparse
import fcntl
import hashlib
import json
import os
import re
import stat
import sys
import uuid
from pathlib import Path
from typing import Any
BASE_ROOT = Path("/home/nvidia/tg3_data_collection")
READY_ROOT = BASE_ROOT / "ready"
DELETING_ROOT = BASE_ROOT / "deleting"
LEDGER_ROOT = BASE_ROOT / "delete_ledger"
SAFE_EPISODE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
SAFE_SHA256 = re.compile(r"^[0-9a-f]{64}$")
MAX_MANIFEST_BYTES = 16 * 1024 * 1024
MAX_LEDGER_BYTES = 64 * 1024
LEDGER_SCHEMA_VERSION = 1
class DeleteSafetyError(RuntimeError):
"""The requested deletion did not satisfy the fixed safety contract."""
class DeleteCollisionError(DeleteSafetyError):
"""A conflicting ready/deleting episode exists and was left untouched."""
def safe_episode_name(value: str) -> bool:
return bool(SAFE_EPISODE_NAME.fullmatch(value)) and value not in (".", "..")
def _open_directory(path: Path) -> int:
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(path, flags)
except OSError as exc:
raise DeleteSafetyError(f"unsafe or missing directory: {path}: {exc}") from exc
try:
metadata = os.fstat(descriptor)
if not stat.S_ISDIR(metadata.st_mode):
raise DeleteSafetyError(f"not a directory: {path}")
return descriptor
except Exception:
os.close(descriptor)
raise
def _open_child_directory(parent_fd: int, name: str) -> int:
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(name, flags, dir_fd=parent_fd)
except OSError as exc:
raise DeleteSafetyError(f"unsafe or missing episode directory: {name}: {exc}") from exc
try:
if not stat.S_ISDIR(os.fstat(descriptor).st_mode):
raise DeleteSafetyError(f"episode is not a directory: {name}")
return descriptor
except Exception:
os.close(descriptor)
raise
def _read_regular_file(parent_fd: int, name: str, *, max_bytes: int) -> bytes:
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(name, flags, dir_fd=parent_fd)
except OSError as exc:
raise DeleteSafetyError(f"missing or unsafe regular file {name!r}: {exc}") from exc
try:
metadata = os.fstat(descriptor)
if not stat.S_ISREG(metadata.st_mode):
raise DeleteSafetyError(f"not a regular file: {name}")
if metadata.st_size > max_bytes:
raise DeleteSafetyError(f"file is too large: {name}")
chunks: list[bytes] = []
remaining = max_bytes + 1
while remaining:
block = os.read(descriptor, min(1024 * 1024, remaining))
if not block:
break
chunks.append(block)
remaining -= len(block)
payload = b"".join(chunks)
if len(payload) > max_bytes:
raise DeleteSafetyError(f"file is too large: {name}")
return payload
finally:
os.close(descriptor)
def _validate_tree_entries(directory_fd: int) -> None:
"""Require the quarantined recorder tree to contain only dirs/files."""
for child in os.listdir(directory_fd):
if child in (".", "..") or "/" in child or "\x00" in child:
raise DeleteSafetyError(f"unsafe directory entry: {child!r}")
metadata = os.stat(child, dir_fd=directory_fd, follow_symlinks=False)
if stat.S_ISDIR(metadata.st_mode):
child_fd = _open_child_directory(directory_fd, child)
try:
_validate_tree_entries(child_fd)
finally:
os.close(child_fd)
elif not stat.S_ISREG(metadata.st_mode):
raise DeleteSafetyError(f"episode contains a symlink/special entry: {child}")
def _ledger_name(episode: str, manifest_sha256: str) -> str:
return f"{episode}.{manifest_sha256}.authorized.json"
def _ledger_payload(
episode: str, manifest_sha256: str, tombstone: str
) -> dict[str, Any]:
return {
"schema_version": LEDGER_SCHEMA_VERSION,
"state": "AUTHORIZED_DELETE",
"episode_id": episode,
"manifest_sha256": manifest_sha256,
"tombstone": tombstone,
}
def _validate_ledger(
ledger_fd: int,
ledger_name: str,
*,
episode: str,
manifest_sha256: str,
tombstone: str,
) -> None:
raw = _read_regular_file(ledger_fd, ledger_name, max_bytes=MAX_LEDGER_BYTES)
try:
payload: Any = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise DeleteSafetyError(f"invalid deletion authorization ledger: {exc}") from exc
if payload != _ledger_payload(episode, manifest_sha256, tombstone):
raise DeleteSafetyError("deletion authorization ledger identity mismatch")
def _write_ledger(
ledger_fd: int,
ledger_name: str,
*,
episode: str,
manifest_sha256: str,
tombstone: str,
) -> None:
payload = (
json.dumps(
_ledger_payload(episode, manifest_sha256, tombstone),
sort_keys=True,
separators=(",", ":"),
)
+ "\n"
).encode("utf-8")
temporary = f".{ledger_name}.{uuid.uuid4().hex}.tmp"
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(temporary, flags, 0o600, dir_fd=ledger_fd)
try:
view = memoryview(payload)
while view:
written = os.write(descriptor, view)
if written <= 0:
raise OSError("short write while creating deletion ledger")
view = view[written:]
os.fsync(descriptor)
finally:
os.close(descriptor)
try:
# A hard-link publish is atomic and refuses to replace any existing
# sidecar. The helper flock serializes legitimate writers as well.
os.link(
temporary,
ledger_name,
src_dir_fd=ledger_fd,
dst_dir_fd=ledger_fd,
follow_symlinks=False,
)
os.fsync(ledger_fd)
finally:
try:
os.unlink(temporary, dir_fd=ledger_fd)
os.fsync(ledger_fd)
except FileNotFoundError:
pass
def _validate_episode(
parent_fd: int,
directory_name: str,
*,
episode: str,
expected_manifest_sha256: str,
) -> None:
episode_fd = _open_child_directory(parent_fd, directory_name)
try:
# READY must be a real regular file. Its contents are deliberately not
# trusted as a deletion credential; the exact manifest digest is.
_read_regular_file(episode_fd, "READY", max_bytes=4096)
raw_manifest = _read_regular_file(
episode_fd, "manifest.json", max_bytes=MAX_MANIFEST_BYTES
)
actual_digest = hashlib.sha256(raw_manifest).hexdigest()
if actual_digest != expected_manifest_sha256:
raise DeleteSafetyError(
"manifest SHA-256 mismatch: "
f"expected {expected_manifest_sha256}, got {actual_digest}"
)
try:
manifest: Any = json.loads(raw_manifest.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise DeleteSafetyError(f"invalid manifest JSON: {exc}") from exc
if not isinstance(manifest, dict):
raise DeleteSafetyError("manifest root is not an object")
if manifest.get("state") != "complete":
raise DeleteSafetyError("manifest state is not complete")
if manifest.get("episode_id") != episode:
raise DeleteSafetyError("manifest episode_id does not match target")
_validate_tree_entries(episode_fd)
finally:
os.close(episode_fd)
def _remove_tree_at(parent_fd: int, name: str) -> None:
"""Remove one child tree without following any symbolic link."""
directory_fd = _open_child_directory(parent_fd, name)
try:
for child in os.listdir(directory_fd):
if child in (".", "..") or "/" in child or "\x00" in child:
raise DeleteSafetyError(f"unsafe directory entry: {child!r}")
metadata = os.stat(child, dir_fd=directory_fd, follow_symlinks=False)
if stat.S_ISDIR(metadata.st_mode):
_remove_tree_at(directory_fd, child)
else:
# Symlinks and all non-directory entries are unlinked, never
# followed. Recorder output is expected to contain only files.
os.unlink(child, dir_fd=directory_fd)
os.fsync(directory_fd)
finally:
os.close(directory_fd)
os.rmdir(name, dir_fd=parent_fd)
def _child_kind(parent_fd: int, name: str) -> str:
try:
metadata = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
except FileNotFoundError:
return "absent"
if stat.S_ISDIR(metadata.st_mode):
return "directory"
if stat.S_ISLNK(metadata.st_mode):
return "symlink"
if stat.S_ISREG(metadata.st_mode):
return "regular"
return "other"
def delete_ready_episode(
episode: str,
manifest_sha256: str,
*,
ready_root: Path = READY_ROOT,
deleting_root: Path = DELETING_ROOT,
ledger_root: Path | None = None,
) -> dict[str, str]:
"""Compare, atomically quarantine, and remove exactly one ready episode.
Root arguments are injectable only for unit tests. The CLI below always
uses the immutable production constants, including a ledger directory that
is outside the recursively removed ``deleting`` tree.
"""
if not safe_episode_name(episode):
raise DeleteSafetyError(f"unsafe episode id: {episode!r}")
if not SAFE_SHA256.fullmatch(manifest_sha256):
raise DeleteSafetyError("manifest SHA-256 must be 64 lowercase hex characters")
ready_root = Path(ready_root)
deleting_root = Path(deleting_root)
ledger_root = Path(ledger_root or (deleting_root.parent / LEDGER_ROOT.name))
if not (
ready_root.parent == deleting_root.parent == ledger_root.parent
):
raise DeleteSafetyError("ready, deleting, and ledger roots must be siblings")
base_root = ready_root.parent
# Refuse a symlinked/wrong parent even in tests. resolve(strict=True) also
# proves that all existing parent components resolve to this exact path.
try:
if base_root.resolve(strict=True) != base_root.absolute():
raise DeleteSafetyError("base root must not contain symlinks")
if ready_root.resolve(strict=True) != ready_root.absolute():
raise DeleteSafetyError("ready root must not contain symlinks")
except FileNotFoundError as exc:
raise DeleteSafetyError(f"fixed recorder root is missing: {exc}") from exc
base_fd = _open_directory(base_root)
ready_fd = _open_directory(ready_root)
deleting_fd: int | None = None
ledger_fd: int | None = None
lock_fd: int | None = None
try:
base_stat = os.fstat(base_fd)
ready_stat = os.fstat(ready_fd)
if base_stat.st_dev != ready_stat.st_dev:
raise DeleteSafetyError("ready root is not on the recorder filesystem")
deleting_kind = _child_kind(base_fd, deleting_root.name)
if deleting_kind == "absent":
os.mkdir(deleting_root.name, mode=0o750, dir_fd=base_fd)
os.fsync(base_fd)
elif deleting_kind != "directory":
raise DeleteSafetyError("deleting root is not a real directory")
deleting_fd = _open_directory(deleting_root)
if os.fstat(deleting_fd).st_dev != ready_stat.st_dev:
raise DeleteSafetyError("deleting root is not on the ready filesystem")
ledger_kind = _child_kind(base_fd, ledger_root.name)
if ledger_kind == "absent":
os.mkdir(ledger_root.name, mode=0o750, dir_fd=base_fd)
os.fsync(base_fd)
elif ledger_kind != "directory":
raise DeleteSafetyError("delete ledger root is not a real directory")
ledger_fd = _open_directory(ledger_root)
if os.fstat(ledger_fd).st_dev != ready_stat.st_dev:
raise DeleteSafetyError("delete ledger is not on the ready filesystem")
lock_flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
lock_fd = os.open(".delete_ready.lock", lock_flags, 0o600, dir_fd=deleting_fd)
if not stat.S_ISREG(os.fstat(lock_fd).st_mode):
raise DeleteSafetyError("deletion lock is not a regular file")
fcntl.flock(lock_fd, fcntl.LOCK_EX)
tombstone = f"{episode}.{manifest_sha256}.deleting"
authorization = _ledger_name(episode, manifest_sha256)
ready_kind = _child_kind(ready_fd, episode)
tombstone_kind = _child_kind(deleting_fd, tombstone)
authorization_kind = _child_kind(ledger_fd, authorization)
# Any other tombstone for this episode is a manifest-identity collision.
prefix = f"{episode}."
collisions = [
name
for name in os.listdir(deleting_fd)
if name.startswith(prefix)
and name.endswith(".deleting")
and name != tombstone
]
if collisions:
raise DeleteCollisionError(
f"conflicting deleting tombstone(s) exist: {sorted(collisions)!r}"
)
ledger_collisions = [
name
for name in os.listdir(ledger_fd)
if name.startswith(prefix)
and name.endswith(".authorized.json")
and name != authorization
]
if ledger_collisions:
raise DeleteCollisionError(
"conflicting deletion authorization(s) exist: "
f"{sorted(ledger_collisions)!r}"
)
if ready_kind not in ("absent", "directory"):
raise DeleteSafetyError(f"ready target is unsafe: {ready_kind}")
if tombstone_kind not in ("absent", "directory"):
raise DeleteSafetyError(f"deleting target is unsafe: {tombstone_kind}")
if authorization_kind not in ("absent", "regular"):
raise DeleteSafetyError(
f"deletion authorization is unsafe: {authorization_kind}"
)
if ready_kind == "directory" and tombstone_kind == "directory":
raise DeleteCollisionError("ready target and deleting tombstone both exist")
if ready_kind == "directory" and authorization_kind == "regular":
raise DeleteCollisionError(
"ready target exists alongside a stale deletion authorization"
)
if ready_kind == "directory":
_validate_episode(
ready_fd,
episode,
episode=episode,
expected_manifest_sha256=manifest_sha256,
)
os.rename(
episode,
tombstone,
src_dir_fd=ready_fd,
dst_dir_fd=deleting_fd,
)
os.fsync(ready_fd)
os.fsync(deleting_fd)
# Re-validate after the rename. If the source name was exchanged
# between the first comparison and rename, the unexpected tree is
# quarantined but is never unlinked.
_validate_episode(
deleting_fd,
tombstone,
episode=episode,
expected_manifest_sha256=manifest_sha256,
)
_write_ledger(
ledger_fd,
authorization,
episode=episode,
manifest_sha256=manifest_sha256,
tombstone=tombstone,
)
authorization_kind = "regular"
result_state = "deleted"
elif tombstone_kind == "directory":
if authorization_kind == "regular":
# The durable out-of-tree ledger remains usable even if an
# interrupted recursive removal already removed READY and the
# manifest from inside the tombstone.
_validate_ledger(
ledger_fd,
authorization,
episode=episode,
manifest_sha256=manifest_sha256,
tombstone=tombstone,
)
else:
# A crash between rename and ledger publication is recoverable
# only while the tombstone still proves its original identity.
_validate_episode(
deleting_fd,
tombstone,
episode=episode,
expected_manifest_sha256=manifest_sha256,
)
_write_ledger(
ledger_fd,
authorization,
episode=episode,
manifest_sha256=manifest_sha256,
tombstone=tombstone,
)
authorization_kind = "regular"
result_state = "resumed_delete"
else:
if authorization_kind == "regular":
# The tree was fully removed and only the final ledger cleanup
# (or its ACK) was interrupted.
_validate_ledger(
ledger_fd,
authorization,
episode=episode,
manifest_sha256=manifest_sha256,
tombstone=tombstone,
)
os.unlink(authorization, dir_fd=ledger_fd)
os.fsync(ledger_fd)
return {
"state": "already_absent",
"episode_id": episode,
"manifest_sha256": manifest_sha256,
}
_remove_tree_at(deleting_fd, tombstone)
os.fsync(deleting_fd)
# Delete authorization is deliberately the final artifact removed.
# If this unlink/ACK is lost, the next invocation validates it and
# safely returns already_absent.
_validate_ledger(
ledger_fd,
authorization,
episode=episode,
manifest_sha256=manifest_sha256,
tombstone=tombstone,
)
os.unlink(authorization, dir_fd=ledger_fd)
os.fsync(ledger_fd)
return {
"state": result_state,
"episode_id": episode,
"manifest_sha256": manifest_sha256,
}
finally:
if lock_fd is not None:
os.close(lock_fd)
if deleting_fd is not None:
os.close(deleting_fd)
if ledger_fd is not None:
os.close(ledger_fd)
os.close(ready_fd)
os.close(base_fd)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("episode_id")
parser.add_argument("manifest_sha256")
return parser.parse_args()
def main() -> int:
args = parse_args()
try:
result = delete_ready_episode(args.episode_id, args.manifest_sha256)
except DeleteCollisionError as exc:
print(json.dumps({"state": "collision", "error": str(exc)}))
return 3
except (DeleteSafetyError, OSError) as exc:
print(json.dumps({"state": "refused", "error": str(exc)}))
return 2
print(json.dumps(result, sort_keys=True))
return 0
if __name__ == "__main__":
sys.exit(main())

221
tg3_local_teleop/test_data_collection.py
View File

@@ -15,12 +15,45 @@ from typing import Any
from data_collection import (
DataRecorderManager,
OptionalTopicGroupConfig,
RecorderConfig,
RecordingToggleGate,
left_joystick_pressed,
)
HEAD_CAMERA_TOPICS = (
"/ob_camera_head/color/image_raw/compressed",
"/ob_camera_head/color/camera_info",
"/ob_camera_head/color/metadata",
"/ob_camera_head/depth/image_raw/compressedDepth",
"/ob_camera_head/depth/camera_info",
"/ob_camera_head/depth/metadata",
)
WAIST_CAMERA_TOPICS = tuple(
topic.replace("_head", "_waist") for topic in HEAD_CAMERA_TOPICS
)
def optional_camera_groups() -> dict[str, OptionalTopicGroupConfig]:
return {
"head_rgbd": OptionalTopicGroupConfig(
topics=HEAD_CAMERA_TOPICS,
minimum_topic_rates_hz={
HEAD_CAMERA_TOPICS[0]: 20.0,
HEAD_CAMERA_TOPICS[3]: 20.0,
},
),
"waist_rgbd": OptionalTopicGroupConfig(
topics=WAIST_CAMERA_TOPICS,
minimum_topic_rates_hz={
WAIST_CAMERA_TOPICS[0]: 20.0,
WAIST_CAMERA_TOPICS[3]: 20.0,
},
),
}
class LeftJoystickParserTest(unittest.TestCase):
def test_accepts_only_live_bool_or_binary_integer(self) -> None:
self.assertIs(
@@ -291,6 +324,10 @@ class DataRecorderManagerTest(unittest.TestCase):
topics: Sequence[str] = ("/joint_states", "/tf"),
required_topics: Sequence[str] = ("/joint_states", "/tf"),
minimum_topic_rates_hz: Mapping[str, float] | None = None,
optional_topic_groups: Mapping[
str, OptionalTopicGroupConfig
] | None = None,
retain_failed_episodes: bool = True,
) -> tuple[DataRecorderManager, Any]:
if factory is None:
factory = FakeProcessFactory(**(process_options or {}))
@@ -305,6 +342,8 @@ class DataRecorderManagerTest(unittest.TestCase):
topics=topics,
required_topics=required_topics,
minimum_topic_rates_hz=minimum_topic_rates_hz or {},
optional_topic_groups=optional_topic_groups or {},
retain_failed_episodes=retain_failed_episodes,
minimum_free_bytes=minimum_free,
max_duration_seconds=max_duration,
poll_interval_seconds=0.005,
@@ -330,6 +369,36 @@ class DataRecorderManagerTest(unittest.TestCase):
self.fail("condition did not become true before timeout")
time.sleep(0.002)
def record_with_optional_cameras(
self, topic_counts: Mapping[str, int]
) -> tuple[dict[str, Any], dict[str, Any]]:
topics = (
"/joint_states",
"/tf",
*HEAD_CAMERA_TOPICS,
*WAIST_CAMERA_TOPICS,
)
manager, _ = self.make_manager(
topics=topics,
required_topics=("/joint_states", "/tf"),
optional_topic_groups=optional_camera_groups(),
process_options={
"topic_counts": topic_counts,
"duration_nanoseconds": 1_000_000_000,
},
)
self.assertTrue(manager.request_start("ca" * 16))
self.wait_for(lambda: manager.status()["recording"])
self.assertTrue(manager.request_stop())
self.assertTrue(manager.wait_until_idle(2.0))
status = manager.status()
manifest = json.loads(
(
Path(status["last_episode_directory"]) / "manifest.json"
).read_text()
)
return status, manifest
def test_manual_stop_uses_sigint_and_atomically_creates_ready_manifest(
self,
) -> None:
@@ -627,6 +696,103 @@ class DataRecorderManagerTest(unittest.TestCase):
0 if head_count is None else head_count,
)
def test_optional_cameras_absent_do_not_block_core_ready(self) -> None:
status, manifest = self.record_with_optional_cameras(
{"/joint_states": 25, "/tf": 50}
)
self.assertEqual(status["last_result"], "ready")
self.assertEqual(manifest["data_quality_warnings"], [])
self.assertEqual(
{
name: observation["state"]
for name, observation in manifest[
"optional_topic_groups"
].items()
},
{"head_rgbd": "absent", "waist_rgbd": "absent"},
)
self.assertEqual(
manifest["optional_topic_groups"]["head_rgbd"][
"below_minimum_rate_topics"
],
[],
)
self.assertTrue((Path(status["last_episode_directory"]) / "READY").exists())
def test_each_active_optional_camera_is_observed_as_healthy(self) -> None:
for active_name, active_topics in (
("head_rgbd", HEAD_CAMERA_TOPICS),
("waist_rgbd", WAIST_CAMERA_TOPICS),
):
with self.subTest(active_name=active_name):
counts = {"/joint_states": 25, "/tf": 50}
counts.update({topic: 1 for topic in active_topics})
counts[active_topics[0]] = 25
counts[active_topics[3]] = 25
status, manifest = self.record_with_optional_cameras(counts)
self.assertEqual(status["last_result"], "ready")
observation = manifest["optional_topic_groups"][active_name]
self.assertEqual(observation["state"], "healthy")
self.assertEqual(
observation["observed_topic_rates_hz"],
{active_topics[0]: 25.0, active_topics[3]: 25.0},
)
inactive_name = (
"waist_rgbd"
if active_name == "head_rgbd"
else "head_rgbd"
)
self.assertEqual(
manifest["optional_topic_groups"][inactive_name]["state"],
"absent",
)
self.assertEqual(manifest["data_quality_warnings"], [])
def test_partial_optional_camera_warns_but_remains_ready(self) -> None:
counts = {"/joint_states": 25, "/tf": 50}
counts.update(
{
HEAD_CAMERA_TOPICS[0]: 25,
HEAD_CAMERA_TOPICS[1]: 1,
HEAD_CAMERA_TOPICS[2]: 1,
}
)
status, manifest = self.record_with_optional_cameras(counts)
self.assertEqual(status["last_result"], "ready")
observation = manifest["optional_topic_groups"]["head_rgbd"]
self.assertEqual(observation["state"], "partial")
self.assertEqual(
observation["zero_message_topics"],
list(HEAD_CAMERA_TOPICS[3:]),
)
self.assertTrue(manifest["data_quality_warnings"])
self.assertIn("is partial", manifest["data_quality_warnings"][0])
self.assertTrue((Path(status["last_episode_directory"]) / "READY").exists())
def test_low_rate_optional_camera_warns_but_remains_ready(self) -> None:
counts = {"/joint_states": 25, "/tf": 50}
counts.update({topic: 1 for topic in WAIST_CAMERA_TOPICS})
counts[WAIST_CAMERA_TOPICS[0]] = 5
counts[WAIST_CAMERA_TOPICS[3]] = 10
status, manifest = self.record_with_optional_cameras(counts)
self.assertEqual(status["last_result"], "ready")
observation = manifest["optional_topic_groups"]["waist_rgbd"]
self.assertEqual(observation["state"], "low_rate")
self.assertEqual(
observation["observed_topic_rates_hz"],
{
WAIST_CAMERA_TOPICS[0]: 5.0,
WAIST_CAMERA_TOPICS[3]: 10.0,
},
)
self.assertEqual(
observation["below_minimum_rate_topics"],
[WAIST_CAMERA_TOPICS[0], WAIST_CAMERA_TOPICS[3]],
)
self.assertTrue(manifest["data_quality_warnings"])
self.assertIn("below", manifest["data_quality_warnings"][0])
self.assertTrue((Path(status["last_episode_directory"]) / "READY").exists())
def test_topic_that_stops_mid_episode_fails_average_rate(self) -> None:
bag_info = FakeBagInfoRunner()
manager, _ = self.make_manager(
@@ -779,6 +945,20 @@ class DataRecorderManagerTest(unittest.TestCase):
)
self.assertEqual(failure["status"], "failed")
def test_failed_payload_is_discarded_when_retention_is_disabled(self) -> None:
manager, factory = self.make_manager(
free_bytes=lambda _path: 0,
retain_failed_episodes=False,
)
self.assertTrue(manager.request_start("d0" * 16))
self.assertTrue(manager.wait_until_idle(2.0))
status = manager.status()
self.assertEqual(status["last_result"], "failed")
self.assertIsNone(status["last_episode_directory"])
self.assertEqual(factory.processes, [])
self.assertEqual(list((self.base / "active").iterdir()), [])
self.assertEqual(list((self.base / "failed").iterdir()), [])
def test_configuration_rejects_unsafe_or_ambiguous_values(self) -> None:
with self.assertRaises(ValueError):
RecorderConfig(self.base, ())
@@ -790,6 +970,8 @@ class DataRecorderManagerTest(unittest.TestCase):
RecorderConfig(self.base, ("/ok",), minimum_free_bytes=-1)
with self.assertRaises(ValueError):
RecorderConfig(self.base, ("/ok",), validate_bag_info=1)
with self.assertRaises(ValueError):
RecorderConfig(self.base, ("/ok",), retain_failed_episodes=1)
with self.assertRaises(ValueError):
RecorderConfig(self.base, ("/ok",), bag_info_timeout_seconds=0)
with self.assertRaises(ValueError):
@@ -827,6 +1009,45 @@ class DataRecorderManagerTest(unittest.TestCase):
required_topics=("/recorded",),
minimum_topic_rates_hz=[], # type: ignore[arg-type]
)
with self.assertRaises(ValueError):
OptionalTopicGroupConfig(topics=())
with self.assertRaises(ValueError):
OptionalTopicGroupConfig(
topics=("/camera/image",),
minimum_topic_rates_hz={"/other": 20.0},
)
camera_group = OptionalTopicGroupConfig(
topics=("/camera/image",),
minimum_topic_rates_hz={"/camera/image": 20.0},
)
with self.assertRaises(ValueError):
RecorderConfig(
self.base,
("/recorded",),
optional_topic_groups={"camera": camera_group},
)
with self.assertRaises(ValueError):
RecorderConfig(
self.base,
("/recorded", "/camera/image"),
required_topics=("/camera/image",),
optional_topic_groups={"camera": camera_group},
)
with self.assertRaises(ValueError):
RecorderConfig(
self.base,
("/recorded", "/camera/image"),
optional_topic_groups={
"camera_a": camera_group,
"camera_b": camera_group,
},
)
with self.assertRaises(ValueError):
RecorderConfig(
self.base,
("/recorded", "/camera/image"),
optional_topic_groups={"bad name": camera_group},
)
if __name__ == "__main__":

376
tg3_local_teleop/test_delete_ready_episode.py Executable file
View File

@@ -0,0 +1,376 @@
#!/usr/bin/env python3
from __future__ import annotations
import hashlib
import json
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from delete_ready_episode import (
DeleteCollisionError,
DeleteSafetyError,
delete_ready_episode,
)
class DeleteReadyEpisodeTests(unittest.TestCase):
def _roots(self, root: Path) -> tuple[Path, Path, Path]:
base = root / "tg3_data_collection"
ready = base / "ready"
deleting = base / "deleting"
active = base / "active"
failed = base / "failed"
for path in (ready, active, failed):
path.mkdir(parents=True, exist_ok=True)
(active / "ACTIVE_SENTINEL").write_text("active", encoding="utf-8")
(failed / "FAILED_SENTINEL").write_text("failed", encoding="utf-8")
return ready, deleting, base
def _episode(
self,
ready: Path,
name: str = "episode_20260810T120000000_deadbeef",
*,
state: str = "complete",
) -> tuple[Path, str]:
episode = ready / name
(episode / "bag").mkdir(parents=True)
(episode / "bag" / "bag_0.mcap").write_bytes(b"mcap")
manifest = {"state": state, "episode_id": name, "files": []}
raw = (json.dumps(manifest, sort_keys=True) + "\n").encode("utf-8")
(episode / "manifest.json").write_bytes(raw)
(episode / "READY").write_text("ready\n", encoding="ascii")
return episode, hashlib.sha256(raw).hexdigest()
def _authorization(
self,
base: Path,
episode: str,
digest: str,
*,
payload_digest: str | None = None,
) -> Path:
ledger = base / "delete_ledger"
ledger.mkdir(exist_ok=True)
tombstone = f"{episode}.{digest}.deleting"
path = ledger / f"{episode}.{digest}.authorized.json"
path.write_text(
json.dumps(
{
"schema_version": 1,
"state": "AUTHORIZED_DELETE",
"episode_id": episode,
"manifest_sha256": payload_digest or digest,
"tombstone": tombstone,
}
),
encoding="utf-8",
)
return path
def test_compare_rename_delete_and_absent_retry_are_idempotent(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
name = episode.name
result = delete_ready_episode(
name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(result["state"], "deleted")
self.assertFalse(episode.exists())
self.assertEqual(
delete_ready_episode(
name,
digest,
ready_root=ready,
deleting_root=deleting,
)["state"],
"already_absent",
)
self.assertEqual(
(base / "active" / "ACTIVE_SENTINEL").read_text(), "active"
)
self.assertEqual(
(base / "failed" / "FAILED_SENTINEL").read_text(), "failed"
)
def test_resume_matching_tombstone_after_lost_ack(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
deleting.mkdir()
tombstone = deleting / f"{episode.name}.{digest}.deleting"
episode.rename(tombstone)
result = delete_ready_episode(
"episode_20260810T120000000_deadbeef",
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(result["state"], "resumed_delete")
self.assertFalse(tombstone.exists())
def test_partial_tree_without_manifest_resumes_from_durable_ledger(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
tombstone = deleting / f"{episode.name}.{digest}.deleting"
ledger = (
base
/ "delete_ledger"
/ f"{episode.name}.{digest}.authorized.json"
)
def interrupt_after_control_files_removed(
_parent_fd: int, _name: str
) -> None:
(tombstone / "READY").unlink()
(tombstone / "manifest.json").unlink()
raise RuntimeError("injected power loss during recursive delete")
with mock.patch(
"delete_ready_episode._remove_tree_at",
side_effect=interrupt_after_control_files_removed,
):
with self.assertRaises(RuntimeError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(tombstone.is_dir())
self.assertFalse((tombstone / "manifest.json").exists())
self.assertTrue(ledger.is_file())
result = delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(result["state"], "resumed_delete")
self.assertFalse(tombstone.exists())
self.assertFalse(ledger.exists())
def test_partial_tombstone_without_ledger_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
deleting.mkdir()
tombstone = deleting / f"{episode.name}.{digest}.deleting"
episode.rename(tombstone)
(tombstone / "manifest.json").unlink()
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(tombstone.exists())
def test_wrong_digest_or_symlink_ledger_is_refused(self) -> None:
for mode in ("wrong_digest", "symlink"):
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
ready, deleting, base = self._roots(root)
episode, digest = self._episode(ready)
deleting.mkdir()
tombstone = deleting / f"{episode.name}.{digest}.deleting"
episode.rename(tombstone)
(tombstone / "manifest.json").unlink()
if mode == "wrong_digest":
ledger = self._authorization(
base, episode.name, digest, payload_digest="f" * 64
)
else:
ledger_root = base / "delete_ledger"
ledger_root.mkdir()
ledger = (
ledger_root
/ f"{episode.name}.{digest}.authorized.json"
)
outside = root / "outside"
outside.write_text("keep", encoding="utf-8")
ledger.symlink_to(outside)
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(tombstone.exists())
self.assertTrue(ledger.exists())
def test_ledger_only_after_tree_removal_is_finalized_idempotently(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, base = self._roots(Path(temporary))
deleting.mkdir()
episode = "episode_20260810T120000000_deadbeef"
digest = "a" * 64
ledger = self._authorization(base, episode, digest)
result = delete_ready_episode(
episode,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(result["state"], "already_absent")
self.assertFalse(ledger.exists())
def test_conflicting_ledger_digest_refuses_ready_target(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
collision = self._authorization(base, episode.name, "b" * 64)
with self.assertRaises(DeleteCollisionError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(episode.exists())
self.assertTrue(collision.exists())
def test_wrong_manifest_digest_refuses_and_preserves_target(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, _digest = self._episode(ready)
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode.name,
"0" * 64,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(episode.exists())
def test_noncomplete_manifest_refuses_and_preserves_target(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, digest = self._episode(ready, state="failed")
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(episode.exists())
def test_unsafe_episode_and_digest_are_rejected(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
for episode, digest in (("../active", "0" * 64), ("safe", "BAD")):
with self.subTest(episode=episode, digest=digest):
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode,
digest,
ready_root=ready,
deleting_root=deleting,
)
def test_symlink_episode_is_refused_without_touching_outside(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
ready, deleting, _base = self._roots(root)
outside = root / "outside"
outside.mkdir()
sentinel = outside / "sentinel"
sentinel.write_text("keep", encoding="utf-8")
name = "episode_20260810T120000000_deadbeef"
(ready / name).symlink_to(outside, target_is_directory=True)
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
name,
"0" * 64,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(sentinel.read_text(), "keep")
def test_symlink_manifest_ready_or_nested_entry_is_refused(self) -> None:
for target in ("manifest", "ready", "nested"):
with self.subTest(target=target), tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
ready, deleting, _base = self._roots(root)
episode, digest = self._episode(ready)
outside = root / "outside"
outside.write_text("keep", encoding="utf-8")
if target == "manifest":
(episode / "manifest.json").unlink()
(episode / "manifest.json").symlink_to(outside)
elif target == "ready":
(episode / "READY").unlink()
(episode / "READY").symlink_to(outside)
else:
(episode / "nested_link").symlink_to(outside)
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertEqual(outside.read_text(), "keep")
def test_conflicting_tombstone_refuses_without_moving_ready(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
deleting.mkdir()
collision = deleting / f"{episode.name}.{'1' * 64}.deleting"
collision.mkdir()
with self.assertRaises(DeleteCollisionError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(episode.exists())
self.assertTrue(collision.exists())
def test_ready_and_same_tombstone_collision_refuses_both(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
ready, deleting, _base = self._roots(Path(temporary))
episode, digest = self._episode(ready)
deleting.mkdir()
tombstone = deleting / f"{episode.name}.{digest}.deleting"
tombstone.mkdir()
with self.assertRaises(DeleteCollisionError):
delete_ready_episode(
episode.name,
digest,
ready_root=ready,
deleting_root=deleting,
)
self.assertTrue(episode.exists())
self.assertTrue(tombstone.exists())
def test_wrong_parent_relationship_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
ready, _deleting, _base = self._roots(root)
elsewhere = root / "elsewhere" / "deleting"
elsewhere.mkdir(parents=True)
with self.assertRaises(DeleteSafetyError):
delete_ready_episode(
"episode_safe",
"0" * 64,
ready_root=ready,
deleting_root=elsewhere,
)
if __name__ == "__main__":
unittest.main()