Organize host and robot streaming releases
This commit is contained in:
Mike Mi
296
robot/v4l2/OmniSocketGo_robot/scripts/BACDauto_test.sh
Normal file
296
robot/v4l2/OmniSocketGo_robot/scripts/BACDauto_test.sh
Normal file
@@ -0,0 +1,296 @@
|
||||
#!/bin/bash
|
||||
|
||||
LOCAL_REPO_DIR="/home/limingjie/LMJ_Work/RobotCompetition/OmniSocketGo"
|
||||
KCP_PEER_BIN="./bin/kcppeer"
|
||||
OUTPUT_ROOT="/home/limingjie/LMJ_Work/RobotCompetition/KCPData/BDAClogs"
|
||||
PEERB_POLL_INTERVAL_SEC=5
|
||||
PEERB_MAX_POLLS=180
|
||||
PEER_A_EXIT_WAIT_SEC=5
|
||||
|
||||
require_local_binary() {
|
||||
if [ ! -x "$1" ]; then
|
||||
echo "ERROR: 缺少可执行文件 $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_remote_peerb() {
|
||||
ssh omni-peer bash -s <<'EOF'
|
||||
pids=$(ps -eo pid=,args= | awk '/[b]in\/kcppeer -id peer-b/ {print $1}')
|
||||
if [ -n "$pids" ]; then
|
||||
kill $pids 2>/dev/null || true
|
||||
fi
|
||||
pids=$(ps -eo pid=,args= | awk '/\/tmp\/peerb_batch\.sh/ {print $1}')
|
||||
if [ -n "$pids" ]; then
|
||||
kill $pids 2>/dev/null || true
|
||||
fi
|
||||
rm -f /tmp/peerb_batch_done /tmp/peerb_batch.sh /tmp/peerb_commands
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "=== 开始自动化测试 ==="
|
||||
|
||||
cd "$LOCAL_REPO_DIR"
|
||||
require_local_binary "$KCP_PEER_BIN"
|
||||
|
||||
echo ">>> 0. 清理上次残留进程..."
|
||||
pkill -f 'bin/kcppeer -id peer-a' 2>/dev/null || true
|
||||
cleanup_remote_peerb || exit 1
|
||||
|
||||
rm -rf logs
|
||||
rm -rf inbox/a
|
||||
mkdir -p logs inbox/a
|
||||
|
||||
# 1. 清理残留 & 启动 Server D 和 Relay C
|
||||
echo ">>> 1. 启动 Server D 和 Relay C..."
|
||||
|
||||
ssh bj-txy bash -s <<'EOF'
|
||||
pkill -f kcpserver 2>/dev/null || true
|
||||
pkill -f 'bin/kcpserver' 2>/dev/null || true
|
||||
sleep 1
|
||||
cd /home/ubuntu/OmniSocketGo
|
||||
rm -rf logs
|
||||
mkdir -p logs
|
||||
if [ ! -x ./bin/kcpserver ]; then
|
||||
echo "ERROR: 缺少 ./bin/kcpserver"
|
||||
exit 1
|
||||
fi
|
||||
setsid ./bin/kcpserver -listen 0.0.0.0:10909 \
|
||||
-telemetry-peer peer-a-telemetry \
|
||||
-kcp-ts-debug-log logs/d-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/d-kcp-stats.jsonl > server_console.log 2>&1 </dev/null &
|
||||
echo "server D launched (pid=$!)"
|
||||
EOF
|
||||
|
||||
ssh sz-txy bash -s <<'EOF'
|
||||
pkill -f kcpserver 2>/dev/null || true
|
||||
pkill -f 'bin/kcpserver' 2>/dev/null || true
|
||||
sleep 1
|
||||
cd /home/ubuntu/OmniSocketGo
|
||||
rm -rf logs
|
||||
mkdir -p logs
|
||||
if [ ! -x ./bin/kcpserver ]; then
|
||||
echo "ERROR: 缺少 ./bin/kcpserver"
|
||||
exit 1
|
||||
fi
|
||||
setsid ./bin/kcpserver -mode=relay -listen 0.0.0.0:10909 -relay-remote 172.21.32.15:10909 > relay_console.log 2>&1 </dev/null &
|
||||
echo "relay C launched (pid=$!)"
|
||||
EOF
|
||||
|
||||
# 等待 server D 端口就绪
|
||||
echo " 等待 Server D 端口就绪..."
|
||||
for i in $(seq 1 60); do
|
||||
if ssh bj-txy "ss -ulnp | grep -q 10909" 2>/dev/null; then
|
||||
echo " Server D 就绪 (${i}s)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo " ERROR: Server D 60s 内未就绪，退出"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 等待 relay C 端口就绪
|
||||
echo " 等待 Relay C 端口就绪..."
|
||||
for i in $(seq 1 60); do
|
||||
if ssh sz-txy "ss -ulnp | grep -q 10909" 2>/dev/null; then
|
||||
echo " Relay C 就绪 (${i}s)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo " ERROR: Relay C 60s 内未就绪，退出"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 2. 启动本地 Peer-A
|
||||
echo ">>> 2. 启动本地 Peer-A..."
|
||||
PEER_A_CMD_FIFO="/tmp/peera_commands_$$"
|
||||
rm -f "$PEER_A_CMD_FIFO"
|
||||
mkfifo "$PEER_A_CMD_FIFO"
|
||||
nohup "$KCP_PEER_BIN" \
|
||||
-id peer-a \
|
||||
-server 172.21.32.15:10909 \
|
||||
-relay-via 106.55.173.235:10909 \
|
||||
-inbox-dir inbox/a \
|
||||
-latency-log logs/a-latency.jsonl \
|
||||
-kcp-ts-debug-log logs/a-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/a-kcp-stats.jsonl \
|
||||
< "$PEER_A_CMD_FIFO" > logs/peera_console.log 2>&1 &
|
||||
PEER_A_PID=$!
|
||||
exec 4>"$PEER_A_CMD_FIFO"
|
||||
|
||||
# 等待 peer-a 注册成功
|
||||
echo " 等待 Peer-A 注册..."
|
||||
for i in $(seq 1 30); do
|
||||
if grep -Eq "opened KCP session as peer-a|connected to .* as peer-a( \\(KCP\\))?" logs/peera_console.log 2>/dev/null; then
|
||||
echo " Peer-A 就绪 (${i}s)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo " WARNING: Peer-A 30s 内未就绪"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 3. 在远端后台启动 peer-b 整个发送流程，不依赖长 SSH 连接
|
||||
echo ">>> 3. 启动远端 Peer-B 并执行 50 轮打流测试..."
|
||||
ssh omni-peer "cd /home/boll/LMJWork/OmniSocketGo && rm -rf logs inbox/b && mkdir -p logs inbox/b"
|
||||
|
||||
PEERB_DONE_FLAG="/tmp/peerb_batch_done"
|
||||
PEERB_BATCH_SCRIPT="/tmp/peerb_batch.sh"
|
||||
|
||||
# 把整个发送脚本写到远端，setsid 后台执行
|
||||
ssh omni-peer bash -s <<'DEPLOY_SCRIPT'
|
||||
DONE_FLAG="/tmp/peerb_batch_done"
|
||||
BATCH_SCRIPT="/tmp/peerb_batch.sh"
|
||||
rm -f "$DONE_FLAG"
|
||||
|
||||
cat > "$BATCH_SCRIPT" <<'INNER_EOF'
|
||||
#!/bin/bash
|
||||
cd /home/boll/LMJWork/OmniSocketGo
|
||||
|
||||
CMD_FIFO=/tmp/peerb_commands
|
||||
DONE_FLAG="/tmp/peerb_batch_done"
|
||||
STATUS="error"
|
||||
rm -f "$CMD_FIFO" "$DONE_FLAG"
|
||||
mkfifo "$CMD_FIFO"
|
||||
|
||||
finish() {
|
||||
local status_to_write="$STATUS"
|
||||
rm -f "$CMD_FIFO"
|
||||
printf '%s\n' "$status_to_write" > "$DONE_FLAG"
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
|
||||
if [ ! -x ./bin/kcppeer ]; then
|
||||
echo "ERROR: 缺少 ./bin/kcppeer" > logs/peerb_console.log
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 启动 peer-b
|
||||
./bin/kcppeer \
|
||||
-id peer-b \
|
||||
-server 81.70.156.140:10909 \
|
||||
-inbox-dir inbox/b \
|
||||
-latency-log logs/b-latency.jsonl \
|
||||
-kcp-ts-debug-log logs/b-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/b-kcp-stats.jsonl \
|
||||
< "$CMD_FIFO" > logs/peerb_console.log 2>&1 &
|
||||
PEER_B_PID=$!
|
||||
|
||||
exec 3>"$CMD_FIFO"
|
||||
|
||||
# 等 peer-b 就绪
|
||||
for i in $(seq 1 60); do
|
||||
if grep -Eq "opened KCP session as peer-b|connected to .* as peer-b( \\(KCP\\))?" logs/peerb_console.log 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 50 轮发送
|
||||
for i in $(seq 1 50); do
|
||||
echo "file peer-a /home/boll/test30.bin" >&3
|
||||
sleep 1
|
||||
echo "file peer-a /home/boll/test5.bin" >&3
|
||||
sleep 1
|
||||
done
|
||||
|
||||
sleep 5
|
||||
echo "quit" >&3 || true
|
||||
exec 3>&-
|
||||
|
||||
peer_b_exited=0
|
||||
for i in $(seq 1 15); do
|
||||
if ! kill -0 $PEER_B_PID 2>/dev/null; then
|
||||
peer_b_exited=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ "$peer_b_exited" -eq 0 ]; then
|
||||
kill $PEER_B_PID 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
if kill -0 $PEER_B_PID 2>/dev/null; then
|
||||
kill -9 $PEER_B_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
wait $PEER_B_PID 2>/dev/null || true
|
||||
|
||||
# 写完成标记
|
||||
STATUS="done"
|
||||
INNER_EOF
|
||||
|
||||
chmod +x "$BATCH_SCRIPT"
|
||||
setsid bash "$BATCH_SCRIPT" </dev/null >/dev/null 2>&1 &
|
||||
echo "peer-b batch launched in background"
|
||||
DEPLOY_SCRIPT
|
||||
|
||||
# 本地轮询等待远端完成（短 SSH 连接，不怕断开）
|
||||
echo " 等待 peer-b 发送完成（预计 ~110 秒）..."
|
||||
for i in $(seq 1 "$PEERB_MAX_POLLS"); do
|
||||
PEERB_STATUS=$(ssh omni-peer "cat /tmp/peerb_batch_done 2>/dev/null || true")
|
||||
if [ "$PEERB_STATUS" = "done" ]; then
|
||||
ELAPSED_SEC=$(( (i - 1) * PEERB_POLL_INTERVAL_SEC ))
|
||||
echo " peer-b 发送完成（约 ${ELAPSED_SEC}s）"
|
||||
break
|
||||
fi
|
||||
if [ "$PEERB_STATUS" = "error" ]; then
|
||||
echo " ERROR: peer-b 后台任务启动失败，请检查远端 logs/peerb_console.log"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$i" -eq "$PEERB_MAX_POLLS" ]; then
|
||||
echo " ERROR: peer-b $((PEERB_MAX_POLLS * PEERB_POLL_INTERVAL_SEC))s 内未完成"
|
||||
exit 1
|
||||
fi
|
||||
# 每 5 秒查一次，减少 SSH 连接频率
|
||||
sleep "$PEERB_POLL_INTERVAL_SEC"
|
||||
done
|
||||
|
||||
# 4. 清理
|
||||
echo ">>> 4. 清理所有进程..."
|
||||
sleep 2
|
||||
echo "quit" >&4 || true
|
||||
exec 4>&-
|
||||
for i in $(seq 1 "$PEER_A_EXIT_WAIT_SEC"); do
|
||||
if ! kill -0 "$PEER_A_PID" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "$PEER_A_PID" 2>/dev/null; then
|
||||
kill "$PEER_A_PID" 2>/dev/null || true
|
||||
fi
|
||||
wait "$PEER_A_PID" 2>/dev/null || true
|
||||
rm -f "$PEER_A_CMD_FIFO"
|
||||
ssh bj-txy "pkill -f kcpserver || true; pkill -f 'bin/kcpserver' || true"
|
||||
ssh sz-txy "pkill -f kcpserver || true; pkill -f 'bin/kcpserver' || true"
|
||||
|
||||
# 获取当前时间戳，格式为 YYYYMMDD_HHMMSS
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
OUTPUT_DIR="$OUTPUT_ROOT/$TIMESTAMP"
|
||||
# 5. 拉取数据 & 生成报告
|
||||
echo ">>> 5. 拉取数据并生成汇总报告..."
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
scp -o ServerAliveInterval=15 -P 10022 boll@175.178.116.187:/home/boll/LMJWork/OmniSocketGo/logs/b-latency.jsonl "$LOCAL_REPO_DIR/logs/b-latency.jsonl" || exit 1
|
||||
|
||||
(cd "$LOCAL_REPO_DIR/go" && go run ./cmd/latencysummary \
|
||||
-input /home/limingjie/LMJ_Work/RobotCompetition/OmniSocketGo/logs/a-latency.jsonl \
|
||||
-input /home/limingjie/LMJ_Work/RobotCompetition/OmniSocketGo/logs/b-latency.jsonl \
|
||||
-output /home/limingjie/LMJ_Work/RobotCompetition/OmniSocketGo/logs/latency-summary.jsonl) || exit 1
|
||||
cd "$LOCAL_REPO_DIR/.." || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/a-latency.jsonl" "$OUTPUT_DIR/a-latency.jsonl" || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/b-latency.jsonl" "$OUTPUT_DIR/b-latency.jsonl" || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/latency-summary.jsonl" "$OUTPUT_DIR/latency-summary.jsonl" || exit 1
|
||||
if [ -f "$LOCAL_REPO_DIR/logs/latency-summary.html" ]; then
|
||||
mv "$LOCAL_REPO_DIR/logs/latency-summary.html" "$OUTPUT_DIR/latency-summary.html" || exit 1
|
||||
fi
|
||||
|
||||
echo "=== 测试完成！==="
|
||||
259
robot/v4l2/OmniSocketGo_robot/scripts/BDAanto_test.sh
Normal file
259
robot/v4l2/OmniSocketGo_robot/scripts/BDAanto_test.sh
Normal file
@@ -0,0 +1,259 @@
|
||||
#!/bin/bash
|
||||
|
||||
LOCAL_REPO_DIR="/home/limingjie/LMJ_Work/RobotCompetition/OmniSocketGo"
|
||||
KCP_PEER_BIN="./bin/kcppeer"
|
||||
OUTPUT_ROOT="/home/limingjie/LMJ_Work/RobotCompetition/KCPData/BCAlogs/"
|
||||
PEERB_POLL_INTERVAL_SEC=5
|
||||
PEERB_MAX_POLLS=180
|
||||
PEER_A_EXIT_WAIT_SEC=5
|
||||
|
||||
require_local_binary() {
|
||||
if [ ! -x "$1" ]; then
|
||||
echo "ERROR: 缺少可执行文件 $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_remote_peerb() {
|
||||
ssh omni-peer bash -s <<'EOF'
|
||||
pids=$(ps -eo pid=,args= | awk '/[b]in\/kcppeer -id peer-b/ {print $1}')
|
||||
if [ -n "$pids" ]; then
|
||||
kill $pids 2>/dev/null || true
|
||||
fi
|
||||
pids=$(ps -eo pid=,args= | awk '/\/tmp\/peerb_batch\.sh/ {print $1}')
|
||||
if [ -n "$pids" ]; then
|
||||
kill $pids 2>/dev/null || true
|
||||
fi
|
||||
rm -f /tmp/peerb_batch_done /tmp/peerb_batch.sh /tmp/peerb_commands
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "=== 开始自动化测试 ==="
|
||||
|
||||
cd "$LOCAL_REPO_DIR"
|
||||
require_local_binary "$KCP_PEER_BIN"
|
||||
|
||||
echo ">>> 0. 清理上次残留进程..."
|
||||
pkill -f 'bin/kcppeer -id peer-a' 2>/dev/null || true
|
||||
cleanup_remote_peerb || exit 1
|
||||
|
||||
rm -rf logs
|
||||
rm -rf inbox/a
|
||||
mkdir -p logs inbox/a
|
||||
|
||||
# 1. 清理残留 & 启动 Server D
|
||||
echo ">>> 1. 启动 Server D..."
|
||||
|
||||
ssh bj-txy bash -s <<'EOF'
|
||||
pkill -f kcpserver 2>/dev/null || true
|
||||
pkill -f 'bin/kcpserver' 2>/dev/null || true
|
||||
sleep 1
|
||||
cd /home/ubuntu/OmniSocketGo
|
||||
rm -rf logs
|
||||
mkdir -p logs
|
||||
if [ ! -x ./bin/kcpserver ]; then
|
||||
echo "ERROR: 缺少 ./bin/kcpserver"
|
||||
exit 1
|
||||
fi
|
||||
setsid ./bin/kcpserver -listen 0.0.0.0:10909 \
|
||||
-telemetry-peer peer-a-telemetry \
|
||||
-kcp-ts-debug-log logs/d-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/d-kcp-stats.jsonl > server_console.log 2>&1 </dev/null &
|
||||
echo "server D launched (pid=$!)"
|
||||
EOF
|
||||
|
||||
# 等待 server D 端口就绪
|
||||
echo " 等待 Server D 端口就绪..."
|
||||
for i in $(seq 1 60); do
|
||||
if ssh bj-txy "ss -ulnp | grep -q 10909" 2>/dev/null; then
|
||||
echo " Server D 就绪 (${i}s)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo " ERROR: Server D 60s 内未就绪，退出"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 2. 启动本地 Peer-A
|
||||
echo ">>> 2. 启动本地 Peer-A..."
|
||||
PEER_A_CMD_FIFO="/tmp/peera_commands_$$"
|
||||
rm -f "$PEER_A_CMD_FIFO"
|
||||
mkfifo "$PEER_A_CMD_FIFO"
|
||||
nohup "$KCP_PEER_BIN" \
|
||||
-id peer-a \
|
||||
-server 81.70.156.140:10909 \
|
||||
-inbox-dir inbox/a \
|
||||
-latency-log logs/a-latency.jsonl \
|
||||
-kcp-ts-debug-log logs/a-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/a-kcp-stats.jsonl \
|
||||
< "$PEER_A_CMD_FIFO" > logs/peera_console.log 2>&1 &
|
||||
PEER_A_PID=$!
|
||||
exec 4>"$PEER_A_CMD_FIFO"
|
||||
|
||||
# 等待 peer-a 注册成功
|
||||
echo " 等待 Peer-A 注册..."
|
||||
for i in $(seq 1 30); do
|
||||
if grep -Eq "opened KCP session as peer-a|connected to .* as peer-a( \\(KCP\\))?" logs/peera_console.log 2>/dev/null; then
|
||||
echo " Peer-A 就绪 (${i}s)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo " WARNING: Peer-A 30s 内未就绪"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 3. 在远端后台启动 peer-b 整个发送流程，不依赖长 SSH 连接
|
||||
echo ">>> 3. 启动远端 Peer-B 并执行 50 轮打流测试..."
|
||||
ssh omni-peer "cd /home/boll/LMJWork/OmniSocketGo && rm -rf logs inbox/b && mkdir -p logs inbox/b"
|
||||
|
||||
PEERB_DONE_FLAG="/tmp/peerb_batch_done"
|
||||
PEERB_BATCH_SCRIPT="/tmp/peerb_batch.sh"
|
||||
|
||||
# 把整个发送脚本写到远端，setsid 后台执行
|
||||
ssh omni-peer bash -s <<'DEPLOY_SCRIPT'
|
||||
DONE_FLAG="/tmp/peerb_batch_done"
|
||||
BATCH_SCRIPT="/tmp/peerb_batch.sh"
|
||||
rm -f "$DONE_FLAG"
|
||||
|
||||
cat > "$BATCH_SCRIPT" <<'INNER_EOF'
|
||||
#!/bin/bash
|
||||
cd /home/boll/LMJWork/OmniSocketGo
|
||||
|
||||
CMD_FIFO=/tmp/peerb_commands
|
||||
DONE_FLAG="/tmp/peerb_batch_done"
|
||||
rm -f "$CMD_FIFO" "$DONE_FLAG"
|
||||
mkfifo "$CMD_FIFO"
|
||||
|
||||
if [ ! -x ./bin/kcppeer ]; then
|
||||
echo "ERROR: 缺少 ./bin/kcppeer" > logs/peerb_console.log
|
||||
echo "error" > "$DONE_FLAG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 启动 peer-b
|
||||
./bin/kcppeer \
|
||||
-id peer-b \
|
||||
-server 81.70.156.140:10909 \
|
||||
-inbox-dir inbox/b \
|
||||
-latency-log logs/b-latency.jsonl \
|
||||
-kcp-ts-debug-log logs/b-kcp-ts.jsonl \
|
||||
-kcp-session-stats-log logs/b-kcp-stats.jsonl \
|
||||
< "$CMD_FIFO" > logs/peerb_console.log 2>&1 &
|
||||
PEER_B_PID=$!
|
||||
|
||||
exec 3>"$CMD_FIFO"
|
||||
|
||||
# 等 peer-b 就绪
|
||||
for i in $(seq 1 60); do
|
||||
if grep -Eq "opened KCP session as peer-b|connected to .* as peer-b( \\(KCP\\))?" logs/peerb_console.log 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 50 轮发送
|
||||
for i in $(seq 1 50); do
|
||||
echo "file peer-a /home/boll/test30.bin" >&3
|
||||
sleep 1
|
||||
echo "file peer-a /home/boll/test5.bin" >&3
|
||||
sleep 1
|
||||
done
|
||||
|
||||
sleep 5
|
||||
echo "quit" >&3
|
||||
exec 3>&-
|
||||
rm -f "$CMD_FIFO"
|
||||
|
||||
peer_b_exited=0
|
||||
for i in $(seq 1 15); do
|
||||
if ! kill -0 $PEER_B_PID 2>/dev/null; then
|
||||
peer_b_exited=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ "$peer_b_exited" -eq 0 ]; then
|
||||
kill $PEER_B_PID 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
if kill -0 $PEER_B_PID 2>/dev/null; then
|
||||
kill -9 $PEER_B_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
wait $PEER_B_PID 2>/dev/null || true
|
||||
|
||||
# 写完成标记
|
||||
echo "done" > "$DONE_FLAG"
|
||||
INNER_EOF
|
||||
|
||||
chmod +x "$BATCH_SCRIPT"
|
||||
setsid bash "$BATCH_SCRIPT" </dev/null >/dev/null 2>&1 &
|
||||
echo "peer-b batch launched in background"
|
||||
DEPLOY_SCRIPT
|
||||
|
||||
# 本地轮询等待远端完成（短 SSH 连接，不怕断开）
|
||||
echo " 等待 peer-b 发送完成（预计 ~110 秒）..."
|
||||
for i in $(seq 1 "$PEERB_MAX_POLLS"); do
|
||||
PEERB_STATUS=$(ssh omni-peer "cat /tmp/peerb_batch_done 2>/dev/null || true")
|
||||
if [ "$PEERB_STATUS" = "done" ]; then
|
||||
ELAPSED_SEC=$(( (i - 1) * PEERB_POLL_INTERVAL_SEC ))
|
||||
echo " peer-b 发送完成（约 ${ELAPSED_SEC}s）"
|
||||
break
|
||||
fi
|
||||
if [ "$PEERB_STATUS" = "error" ]; then
|
||||
echo " ERROR: peer-b 后台任务启动失败，请检查远端 logs/peerb_console.log"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$i" -eq "$PEERB_MAX_POLLS" ]; then
|
||||
echo " ERROR: peer-b $((PEERB_MAX_POLLS * PEERB_POLL_INTERVAL_SEC))s 内未完成"
|
||||
exit 1
|
||||
fi
|
||||
sleep "$PEERB_POLL_INTERVAL_SEC"
|
||||
done
|
||||
|
||||
# 4. 清理
|
||||
echo ">>> 4. 清理所有进程..."
|
||||
sleep 2
|
||||
echo "quit" >&4 || true
|
||||
exec 4>&-
|
||||
for i in $(seq 1 "$PEER_A_EXIT_WAIT_SEC"); do
|
||||
if ! kill -0 "$PEER_A_PID" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "$PEER_A_PID" 2>/dev/null; then
|
||||
kill "$PEER_A_PID" 2>/dev/null || true
|
||||
fi
|
||||
wait "$PEER_A_PID" 2>/dev/null || true
|
||||
rm -f "$PEER_A_CMD_FIFO"
|
||||
ssh bj-txy "pkill -f kcpserver || true; pkill -f 'bin/kcpserver' || true"
|
||||
|
||||
# 获取当前时间戳，格式为 YYYYMMDD_HHMMSS
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
OUTPUT_DIR="$OUTPUT_ROOT/$TIMESTAMP"
|
||||
|
||||
# 5. 拉取数据 & 生成报告
|
||||
echo ">>> 5. 拉取数据并生成汇总报告..."
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
scp -o ServerAliveInterval=15 -P 10022 boll@175.178.116.187:/home/boll/LMJWork/OmniSocketGo/logs/b-latency.jsonl "$LOCAL_REPO_DIR/logs/b-latency.jsonl" || exit 1
|
||||
|
||||
(cd "$LOCAL_REPO_DIR/go" && go run ./cmd/latencysummary \
|
||||
-input "$LOCAL_REPO_DIR/logs/a-latency.jsonl" \
|
||||
-input "$LOCAL_REPO_DIR/logs/b-latency.jsonl" \
|
||||
-output "$LOCAL_REPO_DIR/logs/latency-summary.jsonl") || exit 1
|
||||
|
||||
cd "$LOCAL_REPO_DIR/.." || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/a-latency.jsonl" "$OUTPUT_DIR/a-latency.jsonl" || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/b-latency.jsonl" "$OUTPUT_DIR/b-latency.jsonl" || exit 1
|
||||
mv "$LOCAL_REPO_DIR/logs/latency-summary.jsonl" "$OUTPUT_DIR/latency-summary.jsonl" || exit 1
|
||||
if [ -f "$LOCAL_REPO_DIR/logs/latency-summary.html" ]; then
|
||||
mv "$LOCAL_REPO_DIR/logs/latency-summary.html" "$OUTPUT_DIR/latency-summary.html" || exit 1
|
||||
fi
|
||||
|
||||
echo "=== 测试完成！==="
|
||||
180
robot/v4l2/OmniSocketGo_robot/scripts/boot/5g-dial.sh
Normal file
180
robot/v4l2/OmniSocketGo_robot/scripts/boot/5g-dial.sh
Normal file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="5g-dial"
|
||||
|
||||
append_route_targets() {
|
||||
local raw_list="$1"
|
||||
local target
|
||||
|
||||
if [[ -z "${raw_list}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for target in ${raw_list//,/ }; do
|
||||
if [[ -z "${target}" ]]; then
|
||||
continue
|
||||
fi
|
||||
dial_cmd+=(--route-target "${target}")
|
||||
done
|
||||
}
|
||||
|
||||
read_detected_interface() {
|
||||
local info_json="$1"
|
||||
|
||||
if [[ ! -f "${info_json}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 -c 'import json, sys; print((json.load(open(sys.argv[1], encoding="utf-8")).get("interface") or "").strip())' "${info_json}"
|
||||
}
|
||||
|
||||
disable_interfaces() {
|
||||
local raw_list="$1"
|
||||
local iface
|
||||
local nmcli_available=0
|
||||
|
||||
if [[ -z "${raw_list}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if command -v nmcli >/dev/null 2>&1; then
|
||||
nmcli_available=1
|
||||
fi
|
||||
|
||||
for iface in ${raw_list//,/ }; do
|
||||
if [[ -z "${iface}" ]]; then
|
||||
continue
|
||||
fi
|
||||
blitz_log "${STEP}" "disable-interface" "start" "iface=${iface}" 0
|
||||
if [[ "${nmcli_available}" -eq 1 ]]; then
|
||||
nmcli device disconnect "${iface}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if ip link show dev "${iface}" >/dev/null 2>&1; then
|
||||
if ip link set dev "${iface}" down; then
|
||||
blitz_log "${STEP}" "disable-interface" "success" "iface=${iface}" 0
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "disable-interface" "failure" "iface=${iface}" "${rc}"
|
||||
return "${rc}"
|
||||
fi
|
||||
else
|
||||
blitz_log "${STEP}" "disable-interface" "success" "iface=${iface} not present, skipping" 0
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_serial() {
|
||||
local serial_port="$1"
|
||||
local timeout_sec="$2"
|
||||
local waited=0
|
||||
|
||||
while (( waited < timeout_sec )); do
|
||||
if [[ -e "${serial_port}" ]]; then
|
||||
blitz_log "${STEP}" "wait-serial" "success" "serial_port=${serial_port} waited_sec=${waited}" 0
|
||||
return 0
|
||||
fi
|
||||
if (( waited == 0 || waited % 5 == 0 )); then
|
||||
blitz_log "${STEP}" "wait-serial" "waiting" "serial_port=${serial_port} waited_sec=${waited}" 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$(( waited + 1 ))
|
||||
done
|
||||
|
||||
blitz_log "${STEP}" "wait-serial" "failure" "serial_port=${serial_port} timeout_sec=${timeout_sec}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_route() {
|
||||
local target_ip="$1"
|
||||
local timeout_sec="$2"
|
||||
local expected_interface="${3:-}"
|
||||
local waited=0
|
||||
local route_output
|
||||
|
||||
while (( waited < timeout_sec )); do
|
||||
route_output="$(blitz_route_ready "${target_ip}" "${expected_interface}" || true)"
|
||||
if [[ -n "${route_output}" ]]; then
|
||||
blitz_log "${STEP}" "route-check" "success" "target_ip=${target_ip} interface=${expected_interface:-auto} route=${route_output}" 0
|
||||
return 0
|
||||
fi
|
||||
if (( waited == 0 || waited % 5 == 0 )); then
|
||||
blitz_log "${STEP}" "route-check" "waiting" "target_ip=${target_ip} interface=${expected_interface:-auto} waited_sec=${waited}" 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$(( waited + 1 ))
|
||||
done
|
||||
|
||||
blitz_log "${STEP}" "route-check" "failure" "target_ip=${target_ip} interface=${expected_interface:-auto} timeout_sec=${timeout_sec}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_require_command ip "${STEP}"
|
||||
blitz_require_command python3 "${STEP}"
|
||||
blitz_require_file "${BLITZ_5G_DIAL_DIR}/rndis_dial.py" "${STEP}"
|
||||
|
||||
if [[ -z "${BLITZ_TIME_SERVER_IP}" ]]; then
|
||||
blitz_log "${STEP}" "precheck" "failure" "BLITZ_TIME_SERVER_IP is empty and no fallback could be derived" 1
|
||||
exit 1
|
||||
fi
|
||||
|
||||
disable_interfaces "${BLITZ_5G_DISABLE_INTERFACES:-}"
|
||||
|
||||
if [[ -n "${BLITZ_5G_INTERFACE:-}" ]]; then
|
||||
route_output="$(blitz_route_ready "${BLITZ_TIME_SERVER_IP}" "${BLITZ_5G_INTERFACE}" || true)"
|
||||
if [[ -n "${route_output}" ]]; then
|
||||
blitz_log "${STEP}" "dial" "already_up" "target_ip=${BLITZ_TIME_SERVER_IP} interface=${BLITZ_5G_INTERFACE} route=${route_output}" 0
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
blitz_log "${STEP}" "route-check" "info" "BLITZ_5G_INTERFACE is empty, skipping pre-dial route shortcut and using auto-detect mode" 0
|
||||
fi
|
||||
|
||||
wait_for_serial "${BLITZ_5G_SERIAL_PORT}" "${BLITZ_5G_SERIAL_WAIT_SEC}"
|
||||
|
||||
dial_cmd=(
|
||||
python3
|
||||
rndis_dial.py
|
||||
--serial-port "${BLITZ_5G_SERIAL_PORT}"
|
||||
--modem-subnet "${BLITZ_5G_MODEM_SUBNET}"
|
||||
)
|
||||
if [[ -n "${BLITZ_5G_INTERFACE:-}" ]]; then
|
||||
dial_cmd+=(--interface "${BLITZ_5G_INTERFACE}")
|
||||
fi
|
||||
case "${BLITZ_5G_SKIP_DHCP:-0}" in
|
||||
1|true|TRUE|yes|YES)
|
||||
dial_cmd+=(--skip-dhcp)
|
||||
;;
|
||||
esac
|
||||
case "${BLITZ_5G_REMOVE_DEFAULT_ROUTE:-1}" in
|
||||
1|true|TRUE|yes|YES)
|
||||
dial_cmd+=(--remove-default-route --gateway "${BLITZ_5G_GATEWAY}" --route-target "${BLITZ_TIME_SERVER_IP}")
|
||||
append_route_targets "${BLITZ_5G_ROUTE_TARGETS:-}"
|
||||
;;
|
||||
esac
|
||||
|
||||
pushd "${BLITZ_5G_DIAL_DIR}" >/dev/null
|
||||
blitz_run "${STEP}" "dial" "${dial_cmd[@]}"
|
||||
popd >/dev/null
|
||||
|
||||
resolved_interface="${BLITZ_5G_INTERFACE:-}"
|
||||
if [[ -z "${resolved_interface}" ]]; then
|
||||
resolved_interface="$(read_detected_interface "${BLITZ_5G_INFO_JSON}" || true)"
|
||||
if [[ -n "${resolved_interface}" ]]; then
|
||||
blitz_log "${STEP}" "resolve-interface" "success" "resolved interface from ${BLITZ_5G_INFO_JSON}: ${resolved_interface}" 0
|
||||
else
|
||||
blitz_log "${STEP}" "resolve-interface" "failure" "failed to read detected interface from ${BLITZ_5G_INFO_JSON}" 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${resolved_interface}" ]]; then
|
||||
wait_for_route "${BLITZ_TIME_SERVER_IP}" "${BLITZ_5G_ROUTE_WAIT_SEC}" "${resolved_interface}"
|
||||
blitz_log "${STEP}" "complete" "success" "5G dial completed and route is ready on ${resolved_interface}" 0
|
||||
else
|
||||
blitz_log "${STEP}" "complete" "success" "5G dial completed but route wait was skipped because no interface could be resolved; refer to rndis_dial.py logs" 0
|
||||
fi
|
||||
219
robot/v4l2/OmniSocketGo_robot/scripts/boot/README.md
Normal file
219
robot/v4l2/OmniSocketGo_robot/scripts/boot/README.md
Normal file
@@ -0,0 +1,219 @@
|
||||
# Robot B-Side Boot Chain
|
||||
|
||||
This directory contains the robot-side boot and recovery scripts.
|
||||
|
||||
Normal usage is:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/install-systemd.sh
|
||||
sudo systemctl start blitz-robot.target
|
||||
```
|
||||
|
||||
After installation, `blitz-robot.target` is enabled and will start automatically on reboot.
|
||||
|
||||
To stop the chain now and disable boot-time autostart for future reboots:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/disable-systemd.sh
|
||||
```
|
||||
|
||||
## Current Startup Order
|
||||
|
||||
The current cold-start chain is:
|
||||
|
||||
1. `blitz-boot-gate.service`
|
||||
2. `blitz-5g-dial.service`
|
||||
3. `blitz-ros-receiver.service`
|
||||
4. `blitz-b-side-omnid.service`
|
||||
5. `blitz-watchdog.service`
|
||||
|
||||
There is no longer any automatic time-sync step in the boot chain.
|
||||
|
||||
## What Each Script Does
|
||||
|
||||
- `robot-boot.env`: default boot configuration
|
||||
- `robot-boot.env.local`: machine-local overrides
|
||||
- `common.sh`: shared env loading, logging, and helper functions
|
||||
- `boot-gate.sh`: fixed startup delay gate
|
||||
- `5g-dial.sh`: brings up the 5G modem path and verifies routing
|
||||
- `start-ros-receiver-service.sh`: boot wrapper for ROS receiver
|
||||
- `wait-for-unix-socket.sh`: waits for the ROS receiver unix socket
|
||||
- `start-b-side-omnid-service.sh`: boot wrapper for `b_side_omnid`
|
||||
- `blitz-watchdog.sh`: runtime health watchdog and recovery orchestrator
|
||||
- `blitz-fault-inject.sh`: fault injection entrypoint
|
||||
- `install-systemd.sh`: installs systemd units into `/etc/systemd/system`
|
||||
- `disable-systemd.sh`: stops the boot chain and disables autostart
|
||||
|
||||
## Important Configuration
|
||||
|
||||
Most machine-specific overrides should go into:
|
||||
|
||||
```text
|
||||
scripts/boot/robot-boot.env.local
|
||||
```
|
||||
|
||||
Typical settings:
|
||||
|
||||
```bash
|
||||
BLITZ_BOOT_DELAY_SEC="30"
|
||||
BLITZ_LOG_FILE="/var/log/blitz-robot/startup.log"
|
||||
BLITZ_RUNTIME_DIR="/run/blitz-robot"
|
||||
|
||||
BLITZ_5G_DIAL_DIR="${OMNISOCKETGO_ROOT}/scripts/boot"
|
||||
BLITZ_5G_SERIAL_PORT="/dev/ttyUSB2"
|
||||
BLITZ_5G_INTERFACE=""
|
||||
BLITZ_5G_MODEM_SUBNET="192.168.224.0/22"
|
||||
BLITZ_5G_GATEWAY="192.168.225.1"
|
||||
BLITZ_5G_REMOVE_DEFAULT_ROUTE="1"
|
||||
BLITZ_5G_ROUTE_TARGETS="106.55.173.235"
|
||||
BLITZ_5G_INFO_JSON="${OMNISOCKETGO_ROOT}/scripts/boot/modem_network_info.json"
|
||||
|
||||
BLITZ_TIME_SERVER_IP="81.70.156.140"
|
||||
|
||||
BLITZ_ROS_USER="nvidia"
|
||||
BLITZ_ROS_SOCKET_WAIT_SEC="20"
|
||||
BLITZ_WATCHDOG_INTERVAL_SEC="5"
|
||||
BLITZ_HEALTH_STALE_SEC="15"
|
||||
BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC="15"
|
||||
BLITZ_NETWORK_FAIL_THRESHOLD="3"
|
||||
BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC="30"
|
||||
BLITZ_GPS_MONITOR_ENABLED="1"
|
||||
BLITZ_GPS_DEVICE_GLOB="/dev/ttyCH341USB*"
|
||||
BLITZ_GPS_CHECK_INTERVAL_SEC="10"
|
||||
BLITZ_GPS_RESTART_UNITS="gpsd.socket gpsd.service"
|
||||
BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION="0"
|
||||
```
|
||||
|
||||
`BLITZ_TIME_SERVER_IP` is still used, but only as the 5G route/ping health-check target. It is no longer used for automatic clock synchronization.
|
||||
|
||||
If `BLITZ_TIME_SERVER_IP` is left empty, the scripts fall back to the host part of `ROBOT_SIDE_OMNISOCKET_SERVER_ADDR`.
|
||||
|
||||
## Install Or Upgrade
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/install-systemd.sh
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart blitz-robot.target
|
||||
```
|
||||
|
||||
`install-systemd.sh` will also remove any old `blitz-time-sync.service` unit left over from earlier versions.
|
||||
|
||||
## Disable Autostart
|
||||
|
||||
To stop the currently running services and disable autostart for future reboots:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/disable-systemd.sh
|
||||
```
|
||||
|
||||
To re-enable later:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/install-systemd.sh
|
||||
sudo systemctl start blitz-robot.target
|
||||
```
|
||||
|
||||
## Logs
|
||||
|
||||
All boot-chain and watchdog logs are appended to:
|
||||
|
||||
```text
|
||||
/var/log/blitz-robot/startup.log
|
||||
```
|
||||
|
||||
Follow the log live:
|
||||
|
||||
```bash
|
||||
sudo tail -f /var/log/blitz-robot/startup.log
|
||||
```
|
||||
|
||||
Check service state:
|
||||
|
||||
```bash
|
||||
sudo systemctl status blitz-robot.target
|
||||
sudo systemctl status blitz-5g-dial.service
|
||||
sudo systemctl status blitz-ros-receiver.service
|
||||
sudo systemctl status blitz-b-side-omnid.service
|
||||
sudo systemctl status blitz-watchdog.service
|
||||
```
|
||||
|
||||
Check systemd journal:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u blitz-robot.target -u blitz-5g-dial.service \
|
||||
-u blitz-ros-receiver.service -u blitz-b-side-omnid.service \
|
||||
-u blitz-watchdog.service -f
|
||||
```
|
||||
|
||||
## Runtime Status Files
|
||||
|
||||
The runtime status directory is:
|
||||
|
||||
```text
|
||||
/run/blitz-robot
|
||||
```
|
||||
|
||||
Key files:
|
||||
|
||||
- `b-side-omnid.status.json`
|
||||
- `ros-receiver.status.json`
|
||||
- `watchdog.status.json`
|
||||
|
||||
`watchdog.status.json` now also records `gps_ok` and `gps_device_present` so you can quickly tell whether the GPS USB serial node is currently visible and whether the last `gpsd` reconnect attempt succeeded.
|
||||
|
||||
Pretty-print them:
|
||||
|
||||
```bash
|
||||
sudo python3 -m json.tool /run/blitz-robot/watchdog.status.json
|
||||
sudo python3 -m json.tool /run/blitz-robot/b-side-omnid.status.json
|
||||
sudo python3 -m json.tool /run/blitz-robot/ros-receiver.status.json
|
||||
```
|
||||
|
||||
## Fault Injection
|
||||
|
||||
Available test commands:
|
||||
|
||||
```bash
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh bside-crash
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh bside-process-freeze
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh bside-video-thread-stall
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh bside-control-thread-stall
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh ros-crash
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh ros-freeze
|
||||
```
|
||||
|
||||
For synthetic network fault injection, first enable it in `robot-boot.env.local`:
|
||||
|
||||
```bash
|
||||
BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION="1"
|
||||
```
|
||||
|
||||
Then restart watchdog and inject:
|
||||
|
||||
```bash
|
||||
sudo systemctl restart blitz-watchdog.service
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh network-down on
|
||||
sudo bash scripts/boot/blitz-fault-inject.sh network-down off
|
||||
```
|
||||
|
||||
## Recovery Behavior Summary
|
||||
|
||||
- If `b_side_omnid` dies or its status file goes stale, watchdog first tries a targeted `b_side` restart.
|
||||
- If ROS receiver dies, loses its socket, or its heartbeat goes stale, watchdog performs an ordered full restart:
|
||||
- stop `b_side`
|
||||
- restart ROS receiver
|
||||
- wait for unix socket
|
||||
- start `b_side`
|
||||
- If network checks fail repeatedly, watchdog stops `b_side`, runs `5g-dial.sh`, waits for route recovery, and then restores services.
|
||||
- While 5G is healthy, watchdog keeps every host route listed by `BLITZ_TIME_SERVER_IP` and `BLITZ_5G_ROUTE_TARGETS` pinned to the resolved 5G interface. When 5G becomes unhealthy, watchdog deletes those host routes so traffic can fall back to the remaining default network path. If that fallback path is still reachable, watchdog keeps `b_side_omnid` running instead of treating it as a full network outage.
|
||||
- Whenever watchdog changes or restores those host routes, it logs `route-path` lines for each target so you can see which interface Linux currently chooses for `81.70.156.140`, `106.55.173.235`, and any other configured 5G-pinned target.
|
||||
- If GPS monitoring is enabled, watchdog checks `BLITZ_GPS_DEVICE_GLOB` every `BLITZ_GPS_CHECK_INTERVAL_SEC` seconds. When the GPS serial device disappears and later reappears, watchdog restarts the units in `BLITZ_GPS_RESTART_UNITS` so `gpsd` can bind to the new device node again.
|
||||
- Camera disappearance is logged as degraded state. Reappearance triggers a `b_side` restart after the device is stable.
|
||||
|
||||
## Notes
|
||||
|
||||
- `time-sync.sh` and `blitz-time-sync.service` are intentionally removed from the automatic boot path.
|
||||
- `b_side_omnid` must already be built before boot-time startup.
|
||||
- `bin/b_side_omnid` missing, ROS env missing, or modem script missing will all show up in `startup.log`.
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="5g-link-logger"
|
||||
|
||||
resolve_target_ip() {
|
||||
if [[ -n "${BLITZ_TIME_SERVER_IP:-}" ]]; then
|
||||
printf '%s\n' "${BLITZ_TIME_SERVER_IP}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for candidate in ${BLITZ_5G_ROUTE_TARGETS//,/ }; do
|
||||
if [[ -n "${candidate}" ]]; then
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
emit_sample_json() {
|
||||
local interface_name="${1:-}"
|
||||
local target_ip="${2:-}"
|
||||
|
||||
python3 - "${interface_name}" "${target_ip}" <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
interface_name = sys.argv[1]
|
||||
target_ip = sys.argv[2]
|
||||
|
||||
payload = {
|
||||
"ts_unix_ms": time.time_ns() // 1_000_000,
|
||||
"interface": interface_name,
|
||||
"target_ip": target_ip,
|
||||
"link_present": False,
|
||||
"route_output": "",
|
||||
"route_ok": False,
|
||||
"probe_ok": False,
|
||||
"ping_rtt_ms": None,
|
||||
"rx_bytes": 0,
|
||||
"tx_bytes": 0,
|
||||
"rx_packets": 0,
|
||||
"tx_packets": 0,
|
||||
"rx_errors": 0,
|
||||
"tx_errors": 0,
|
||||
"rx_drops": 0,
|
||||
"tx_drops": 0,
|
||||
}
|
||||
|
||||
if interface_name:
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
["ip", "-j", "-s", "link", "show", "dev", interface_name],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
stats = json.loads(output)
|
||||
if stats:
|
||||
item = stats[0]
|
||||
payload["link_present"] = True
|
||||
rx = item.get("stats64", {}).get("rx", {})
|
||||
tx = item.get("stats64", {}).get("tx", {})
|
||||
if not rx and not tx:
|
||||
rx = item.get("stats", {}).get("rx", {})
|
||||
tx = item.get("stats", {}).get("tx", {})
|
||||
payload["rx_bytes"] = int(rx.get("bytes") or 0)
|
||||
payload["tx_bytes"] = int(tx.get("bytes") or 0)
|
||||
payload["rx_packets"] = int(rx.get("packets") or 0)
|
||||
payload["tx_packets"] = int(tx.get("packets") or 0)
|
||||
payload["rx_errors"] = int(rx.get("errors") or 0)
|
||||
payload["tx_errors"] = int(tx.get("errors") or 0)
|
||||
payload["rx_drops"] = int(rx.get("dropped") or 0)
|
||||
payload["tx_drops"] = int(tx.get("dropped") or 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if target_ip:
|
||||
try:
|
||||
route = subprocess.check_output(
|
||||
["ip", "route", "get", target_ip],
|
||||
text=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
).strip()
|
||||
payload["route_output"] = route.splitlines()[0] if route else ""
|
||||
payload["route_ok"] = bool(payload["route_output"]) and (
|
||||
not interface_name or f" dev {interface_name}" in payload["route_output"]
|
||||
)
|
||||
except Exception as exc:
|
||||
payload["route_output"] = str(exc)
|
||||
|
||||
ping_cmd = ["ping", "-c", "1", "-W", "2", target_ip]
|
||||
if interface_name:
|
||||
ping_cmd[1:1] = ["-I", interface_name]
|
||||
ping = subprocess.run(ping_cmd, capture_output=True, text=True)
|
||||
payload["probe_ok"] = ping.returncode == 0
|
||||
output = (ping.stdout or "") + "\n" + (ping.stderr or "")
|
||||
for token in output.replace("\n", " ").split():
|
||||
if token.startswith("time="):
|
||||
value = token.split("=", 1)[1].rstrip("ms")
|
||||
try:
|
||||
payload["ping_rtt_ms"] = float(value)
|
||||
except ValueError:
|
||||
pass
|
||||
break
|
||||
|
||||
print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
if [[ "${OMNI_BOOT_MODE:-0}" == "1" ]]; then
|
||||
blitz_load_boot_env
|
||||
blitz_require_run_context
|
||||
fi
|
||||
|
||||
if [[ -z "${BLITZ_RUN_DIR:-}" && -f "${BLITZ_RUN_CONTEXT_FILE:-}" ]]; then
|
||||
blitz_load_run_context_env || true
|
||||
fi
|
||||
blitz_ensure_instance_id
|
||||
|
||||
export BLITZ_5G_LINK_LOG_PATH="${BLITZ_5G_LINK_LOG_PATH:-${BLITZ_RUN_DIR}/b-5g-link-quality.${BLITZ_INSTANCE_ID}.jsonl}"
|
||||
target_ip="$(resolve_target_ip || true)"
|
||||
|
||||
blitz_log "${STEP}" "start" "start" "path=${BLITZ_5G_LINK_LOG_PATH} interval_sec=${BLITZ_5G_LINK_LOG_INTERVAL_SEC}" 0
|
||||
|
||||
while true; do
|
||||
interface_name="$(blitz_resolve_5g_interface || true)"
|
||||
line="$(emit_sample_json "${interface_name}" "${target_ip}")"
|
||||
blitz_jsonl_append_line "${BLITZ_5G_LINK_LOG_PATH}" "${line}"
|
||||
sleep "${BLITZ_5G_LINK_LOG_INTERVAL_SEC}"
|
||||
done
|
||||
139
robot/v4l2/OmniSocketGo_robot/scripts/boot/blitz-fault-inject.sh
Normal file
139
robot/v4l2/OmniSocketGo_robot/scripts/boot/blitz-fault-inject.sh
Normal file
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="fault-inject"
|
||||
B_SIDE_SERVICE="blitz-b-side-omnid.service"
|
||||
ROS_SERVICE="blitz-ros-receiver.service"
|
||||
|
||||
main_pid_for_service() {
|
||||
local service_name="$1"
|
||||
systemctl show --property MainPID --value "${service_name}"
|
||||
}
|
||||
|
||||
wait_for_service_pid_change() {
|
||||
local service_name="$1"
|
||||
local previous_pid="$2"
|
||||
local timeout_sec="${3:-10}"
|
||||
local waited=0
|
||||
local current_pid=""
|
||||
|
||||
while (( waited < timeout_sec )); do
|
||||
current_pid="$(main_pid_for_service "${service_name}")"
|
||||
if [[ -n "${current_pid}" && "${current_pid}" != "0" && "${current_pid}" != "${previous_pid}" ]]; then
|
||||
printf '%s\n' "${current_pid}"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$(( waited + 1 ))
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
require_running_pid() {
|
||||
local service_name="$1"
|
||||
local pid
|
||||
|
||||
pid="$(main_pid_for_service "${service_name}")"
|
||||
if [[ -z "${pid}" || "${pid}" == "0" ]]; then
|
||||
blitz_log "${STEP}" "lookup-pid" "failure" "service=${service_name}" 1
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "${pid}"
|
||||
}
|
||||
|
||||
write_fault_flag() {
|
||||
local flag_name="$1"
|
||||
local flag_path="${BLITZ_RUNTIME_DIR}/${flag_name}"
|
||||
printf '%s\n' "$(date +%s)" > "${flag_path}"
|
||||
blitz_log "${STEP}" "flag-on" "success" "path=${flag_path}" 0
|
||||
}
|
||||
|
||||
clear_fault_flag() {
|
||||
local flag_name="$1"
|
||||
local flag_path="${BLITZ_RUNTIME_DIR}/${flag_name}"
|
||||
rm -f "${flag_path}"
|
||||
blitz_log "${STEP}" "flag-off" "success" "path=${flag_path}" 0
|
||||
}
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_prepare_runtime_dir
|
||||
|
||||
case "${1:-}" in
|
||||
bside-crash)
|
||||
target_pid="$(require_running_pid "${B_SIDE_SERVICE}")"
|
||||
blitz_log "${STEP}" "bside-crash" "start" "service=${B_SIDE_SERVICE} pid=${target_pid}" 0
|
||||
kill -9 "${target_pid}"
|
||||
if restarted_pid="$(wait_for_service_pid_change "${B_SIDE_SERVICE}" "${target_pid}")"; then
|
||||
blitz_log "${STEP}" "bside-crash" "success" "old_pid=${target_pid} new_pid=${restarted_pid}" 0
|
||||
else
|
||||
blitz_log "${STEP}" "bside-crash" "failure" "old_pid=${target_pid} restart_not_observed_within=10s" 1
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
bside-process-freeze)
|
||||
target_pid="$(require_running_pid "${B_SIDE_SERVICE}")"
|
||||
blitz_log "${STEP}" "bside-process-freeze" "start" "service=${B_SIDE_SERVICE} pid=${target_pid}" 0
|
||||
kill -STOP "${target_pid}"
|
||||
blitz_log "${STEP}" "bside-process-freeze" "success" "service=${B_SIDE_SERVICE} pid=${target_pid}" 0
|
||||
;;
|
||||
bside-video-thread-stall)
|
||||
write_fault_flag "fault-injection-bside-video-thread-stall"
|
||||
;;
|
||||
bside-control-thread-stall)
|
||||
write_fault_flag "fault-injection-bside-control-thread-stall"
|
||||
;;
|
||||
ros-crash)
|
||||
target_pid="$(require_running_pid "${ROS_SERVICE}")"
|
||||
blitz_log "${STEP}" "ros-crash" "start" "service=${ROS_SERVICE} pid=${target_pid}" 0
|
||||
kill -9 "${target_pid}"
|
||||
if restarted_pid="$(wait_for_service_pid_change "${ROS_SERVICE}" "${target_pid}")"; then
|
||||
blitz_log "${STEP}" "ros-crash" "success" "old_pid=${target_pid} new_pid=${restarted_pid}" 0
|
||||
else
|
||||
blitz_log "${STEP}" "ros-crash" "failure" "old_pid=${target_pid} restart_not_observed_within=10s" 1
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
ros-freeze)
|
||||
target_pid="$(require_running_pid "${ROS_SERVICE}")"
|
||||
blitz_log "${STEP}" "ros-freeze" "start" "service=${ROS_SERVICE} pid=${target_pid}" 0
|
||||
kill -STOP "${target_pid}"
|
||||
blitz_log "${STEP}" "ros-freeze" "success" "service=${ROS_SERVICE} pid=${target_pid}" 0
|
||||
;;
|
||||
network-down)
|
||||
if [[ "${BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION}" != "1" ]]; then
|
||||
blitz_log "${STEP}" "network-down" "failure" "set BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION=1 first" 1
|
||||
exit 1
|
||||
fi
|
||||
case "${2:-}" in
|
||||
on)
|
||||
write_fault_flag "fault-injection-network-down"
|
||||
;;
|
||||
off)
|
||||
clear_fault_flag "fault-injection-network-down"
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 network-down on|off" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
cat <<'EOF'
|
||||
usage:
|
||||
blitz-fault-inject.sh bside-crash
|
||||
blitz-fault-inject.sh bside-process-freeze
|
||||
blitz-fault-inject.sh bside-video-thread-stall
|
||||
blitz-fault-inject.sh bside-control-thread-stall
|
||||
blitz-fault-inject.sh ros-crash
|
||||
blitz-fault-inject.sh ros-freeze
|
||||
blitz-fault-inject.sh network-down on|off
|
||||
EOF
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="incident-launch"
|
||||
incident_id=""
|
||||
args=()
|
||||
timeout_bin=""
|
||||
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--incident-id)
|
||||
incident_id="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
args+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_require_command systemd-run "${STEP}"
|
||||
blitz_require_command timeout "${STEP}"
|
||||
timeout_bin="$(command -v timeout)"
|
||||
|
||||
if [[ -z "${incident_id}" ]]; then
|
||||
incident_id="$(blitz_new_incident_id)"
|
||||
fi
|
||||
|
||||
unit_name="blitz-incident-${incident_id//[^A-Za-z0-9_.-]/-}"
|
||||
|
||||
systemd-run \
|
||||
--quiet \
|
||||
--collect \
|
||||
--unit "${unit_name}" \
|
||||
--property=Type=oneshot \
|
||||
--property="StandardOutput=append:${BLITZ_LOG_FILE}" \
|
||||
--property="StandardError=append:${BLITZ_LOG_FILE}" \
|
||||
"${timeout_bin}" "${BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC}s" \
|
||||
/bin/bash "${SCRIPT_DIR}/blitz-incident-capture.sh" \
|
||||
--incident-id "${incident_id}" \
|
||||
"${args[@]}"
|
||||
|
||||
printf '%s\n' "${incident_id}"
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="incident-capture"
|
||||
incident_id=""
|
||||
incident_source=""
|
||||
incident_reason=""
|
||||
incident_unit=""
|
||||
incident_result=""
|
||||
incident_exit_status=""
|
||||
|
||||
run_capture() {
|
||||
local output_path="$1"
|
||||
shift
|
||||
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
timeout "${BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC}s" "$@" > "${output_path}" 2>&1 || true
|
||||
else
|
||||
"$@" > "${output_path}" 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--incident-id)
|
||||
incident_id="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--source)
|
||||
incident_source="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--reason)
|
||||
incident_reason="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--unit)
|
||||
incident_unit="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--result)
|
||||
incident_result="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--exit-status)
|
||||
incident_exit_status="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
blitz_log "${STEP}" "parse-arg" "failure" "unknown argument: $1" 2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "${incident_result}" && "${incident_result}" == "success" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_load_run_context_env || true
|
||||
blitz_prepare_runtime_dir
|
||||
blitz_prepare_run_root
|
||||
|
||||
if [[ -z "${incident_id}" ]]; then
|
||||
incident_id="$(blitz_new_incident_id)"
|
||||
fi
|
||||
|
||||
incident_dir="${BLITZ_RUN_ROOT}/incidents/${incident_id}"
|
||||
mkdir -p "${incident_dir}"
|
||||
|
||||
python3 - "${incident_dir}/incident.json" "${incident_id}" "${BLITZ_RUN_ID:-}" "${incident_source}" "${incident_reason}" "${incident_unit}" "${incident_result}" "${incident_exit_status}" "${BLITZ_RUN_DIR:-}" "${HOSTNAME:-$(hostname)}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
path, incident_id, run_id, source, reason, unit, result, exit_status, run_dir, hostname = sys.argv[1:10]
|
||||
payload = {
|
||||
"incident_id": incident_id,
|
||||
"run_id": run_id,
|
||||
"source": source,
|
||||
"fault_reason": reason,
|
||||
"unit": unit,
|
||||
"service_result": result,
|
||||
"exit_status": exit_status,
|
||||
"run_dir": run_dir,
|
||||
"hostname": hostname,
|
||||
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
PY
|
||||
|
||||
for status_file in \
|
||||
"${BLITZ_RUNTIME_DIR}/watchdog.status.json" \
|
||||
"${BLITZ_RUNTIME_DIR}/b-side-omnid.status.json" \
|
||||
"${BLITZ_RUNTIME_DIR}/ros-receiver.status.json"
|
||||
do
|
||||
if [[ -f "${status_file}" ]]; then
|
||||
cp -f "${status_file}" "${incident_dir}/$(basename "${status_file}")"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -f "${BLITZ_LOG_FILE}" ]]; then
|
||||
tail -n 400 "${BLITZ_LOG_FILE}" > "${incident_dir}/startup.log.tail"
|
||||
fi
|
||||
|
||||
run_capture "${incident_dir}/systemctl-status.txt" \
|
||||
systemctl status blitz-robot.target blitz-run-context.service blitz-5g-dial.service blitz-5g-link-logger.service blitz-ros-receiver.service blitz-b-side-omnid.service blitz-watchdog.service
|
||||
run_capture "${incident_dir}/journal.txt" \
|
||||
journalctl --no-pager --since "5 minutes ago" -u blitz-run-context.service -u blitz-5g-dial.service -u blitz-5g-link-logger.service -u blitz-ros-receiver.service -u blitz-b-side-omnid.service -u blitz-watchdog.service
|
||||
run_capture "${incident_dir}/ip-addr.txt" ip addr
|
||||
run_capture "${incident_dir}/ip-route.txt" ip route
|
||||
run_capture "${incident_dir}/ss-uapn.txt" ss -uapn
|
||||
run_capture "${incident_dir}/ss-xlp.txt" ss -xlp
|
||||
|
||||
if [[ -f "${BLITZ_5G_INFO_JSON:-}" ]]; then
|
||||
cp -f "${BLITZ_5G_INFO_JSON}" "${incident_dir}/$(basename "${BLITZ_5G_INFO_JSON}")"
|
||||
fi
|
||||
|
||||
if [[ -n "${BLITZ_RUN_DIR:-}" && -d "${BLITZ_RUN_DIR}" ]]; then
|
||||
while IFS= read -r -d '' jsonl; do
|
||||
tail -n 200 "${jsonl}" > "${incident_dir}/tail-$(basename "${jsonl}")"
|
||||
done < <(find "${BLITZ_RUN_DIR}" -maxdepth 1 -type f -name '*.jsonl' -print0 2>/dev/null)
|
||||
fi
|
||||
|
||||
blitz_log "${STEP}" "complete" "success" "incident_id=${incident_id} path=${incident_dir}" 0
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="run-context"
|
||||
|
||||
on_error() {
|
||||
local rc="$?"
|
||||
blitz_log "${STEP}" "error" "failure" "line=${1:-unknown} cmd=${BASH_COMMAND:-unknown}" "${rc}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
trap 'on_error "${LINENO}"' ERR
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_require_command python3 "${STEP}"
|
||||
blitz_init_run_context
|
||||
blitz_log "${STEP}" "complete" "success" "run_id=${BLITZ_RUN_ID} run_dir=${BLITZ_RUN_DIR}" 0
|
||||
971
robot/v4l2/OmniSocketGo_robot/scripts/boot/blitz-watchdog.sh
Normal file
971
robot/v4l2/OmniSocketGo_robot/scripts/boot/blitz-watchdog.sh
Normal file
@@ -0,0 +1,971 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="watchdog"
|
||||
B_SIDE_SERVICE="blitz-b-side-omnid.service"
|
||||
ROS_SERVICE="blitz-ros-receiver.service"
|
||||
B_SIDE_STATUS_FILE=""
|
||||
ROS_STATUS_FILE=""
|
||||
WATCHDOG_STATUS_FILE=""
|
||||
NETWORK_FAULT_FILE=""
|
||||
WATCHDOG_EVENT_LOG=""
|
||||
WATCHDOG_SAMPLE_LOG=""
|
||||
WATCHDOG_EVENT_LOG_FAILURE_REPORTED=0
|
||||
WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED=0
|
||||
CAMERA_MISSING_PREV=0
|
||||
CAMERA_RECOVERY_STABLE_COUNT=0
|
||||
NETWORK_FAIL_COUNT=0
|
||||
NETWORK_COOLDOWN_UNTIL=0
|
||||
BACKOFF_UNTIL=0
|
||||
LAST_ACTION="none"
|
||||
LAST_ACTION_EPOCH_MS=0
|
||||
FULL_RESTART_WINDOW_START=0
|
||||
FULL_RESTART_WINDOW_COUNT=0
|
||||
NETWORK_LAST_INTERFACE=""
|
||||
NETWORK_ROUTE_INTERFACE_LAST_KNOWN=""
|
||||
NETWORK_PRIMARY_LAST_RETRY_SEC=0
|
||||
GPS_LAST_CHECK_SEC=0
|
||||
GPS_DEVICE_PRESENT_PREV=-1
|
||||
GPS_DEVICE_PRESENT_STATE=1
|
||||
GPS_STACK_ACTIVE_STATE=1
|
||||
LAST_REPORTED_FAULT_REASON=""
|
||||
LAST_REPORTED_RECOVERY_STATE=""
|
||||
declare -A TARGETED_RESTART_WINDOW_START=()
|
||||
declare -A TARGETED_RESTART_WINDOW_COUNT=()
|
||||
|
||||
now_epoch_sec() {
|
||||
date +%s
|
||||
}
|
||||
|
||||
now_epoch_ms() {
|
||||
date +%s%3N
|
||||
}
|
||||
|
||||
service_is_active() {
|
||||
systemctl is-active --quiet "$1"
|
||||
}
|
||||
|
||||
gps_monitor_enabled() {
|
||||
[[ "${BLITZ_GPS_MONITOR_ENABLED:-0}" == "1" ]]
|
||||
}
|
||||
|
||||
gps_stack_active() {
|
||||
local units=()
|
||||
local unit
|
||||
|
||||
read -r -a units <<< "${BLITZ_GPS_RESTART_UNITS:-}"
|
||||
if (( ${#units[@]} == 0 )); then
|
||||
return 1
|
||||
fi
|
||||
|
||||
for unit in "${units[@]}"; do
|
||||
if service_is_active "${unit}"; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_gps_stack() {
|
||||
local reason="$1"
|
||||
local devices="$2"
|
||||
local units=()
|
||||
local rc
|
||||
|
||||
read -r -a units <<< "${BLITZ_GPS_RESTART_UNITS:-}"
|
||||
if (( ${#units[@]} == 0 )); then
|
||||
GPS_STACK_ACTIVE_STATE=0
|
||||
blitz_log "${STEP}" "gps-reconnect" "failure" "reason=${reason} devices=${devices} units=empty" 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
set_last_action "gps-reconnect"
|
||||
blitz_log "${STEP}" "gps-reconnect" "start" "reason=${reason} devices=${devices} units=${BLITZ_GPS_RESTART_UNITS}" 0
|
||||
if systemctl restart "${units[@]}"; then
|
||||
GPS_STACK_ACTIVE_STATE=1
|
||||
blitz_log "${STEP}" "gps-reconnect" "success" "reason=${reason} devices=${devices} units=${BLITZ_GPS_RESTART_UNITS}" 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
rc=$?
|
||||
GPS_STACK_ACTIVE_STATE=0
|
||||
blitz_log "${STEP}" "gps-reconnect" "failure" "reason=${reason} devices=${devices} units=${BLITZ_GPS_RESTART_UNITS}" "${rc}"
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
check_gps_health() {
|
||||
local now_sec="$1"
|
||||
local check_interval_sec="${BLITZ_GPS_CHECK_INTERVAL_SEC:-10}"
|
||||
local device_glob="${BLITZ_GPS_DEVICE_GLOB:-}"
|
||||
local previous_present="${GPS_DEVICE_PRESENT_PREV}"
|
||||
local recovery_reason=""
|
||||
local device_summary=""
|
||||
local -a devices=()
|
||||
|
||||
if ! gps_monitor_enabled; then
|
||||
GPS_DEVICE_PRESENT_STATE=1
|
||||
GPS_STACK_ACTIVE_STATE=1
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( check_interval_sec < 1 )); then
|
||||
check_interval_sec=1
|
||||
fi
|
||||
if (( GPS_LAST_CHECK_SEC != 0 && now_sec - GPS_LAST_CHECK_SEC < check_interval_sec )); then
|
||||
if (( GPS_DEVICE_PRESENT_STATE == 1 && GPS_STACK_ACTIVE_STATE == 1 )); then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
GPS_LAST_CHECK_SEC="${now_sec}"
|
||||
|
||||
mapfile -t devices < <(compgen -G "${device_glob}" || true)
|
||||
if (( ${#devices[@]} == 0 )); then
|
||||
GPS_DEVICE_PRESENT_STATE=0
|
||||
GPS_STACK_ACTIVE_STATE=0
|
||||
if (( previous_present != 0 )); then
|
||||
blitz_log "${STEP}" "gps-device-check" "failure" "state=missing glob=${device_glob}" 1
|
||||
fi
|
||||
GPS_DEVICE_PRESENT_PREV=0
|
||||
return 1
|
||||
fi
|
||||
|
||||
device_summary="$(IFS=,; printf '%s' "${devices[*]}")"
|
||||
GPS_DEVICE_PRESENT_STATE=1
|
||||
GPS_DEVICE_PRESENT_PREV=1
|
||||
|
||||
if (( previous_present == 0 )); then
|
||||
blitz_log "${STEP}" "gps-device-check" "success" "state=reappeared devices=${device_summary}" 0
|
||||
recovery_reason="device-reappeared"
|
||||
elif ! gps_stack_active; then
|
||||
recovery_reason="gpsd-inactive"
|
||||
fi
|
||||
|
||||
if [[ -n "${recovery_reason}" ]]; then
|
||||
if restart_gps_stack "${recovery_reason}" "${device_summary}"; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
GPS_STACK_ACTIVE_STATE=1
|
||||
return 0
|
||||
}
|
||||
|
||||
status_file_fresh() {
|
||||
local path="$1"
|
||||
local max_age_sec="$2"
|
||||
local now_sec
|
||||
local mtime_sec
|
||||
|
||||
if [[ ! -f "${path}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
now_sec="$(now_epoch_sec)"
|
||||
mtime_sec="$(stat -c %Y "${path}" 2>/dev/null || echo 0)"
|
||||
(( now_sec - mtime_sec <= max_age_sec ))
|
||||
}
|
||||
|
||||
ros_receiver_status_fresh() {
|
||||
local path="$1"
|
||||
local max_age_sec="$2"
|
||||
local now_epoch_ms_value
|
||||
|
||||
now_epoch_ms_value="$(now_epoch_ms)"
|
||||
python3 - "${path}" "${now_epoch_ms_value}" "${max_age_sec}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
now_epoch_ms = int(sys.argv[2])
|
||||
max_age_ms = int(sys.argv[3]) * 1000
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except Exception:
|
||||
raise SystemExit(1)
|
||||
|
||||
heartbeat_ms = int(payload.get("recv_thread_heartbeat_epoch_ms") or 0)
|
||||
socket_bound = bool(payload.get("socket_bound"))
|
||||
|
||||
if heartbeat_ms <= 0 or not socket_bound:
|
||||
raise SystemExit(1)
|
||||
|
||||
raise SystemExit(0 if now_epoch_ms - heartbeat_ms <= max_age_ms else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
ros_receiver_healthy() {
|
||||
local max_age_sec="$1"
|
||||
|
||||
service_is_active "${ROS_SERVICE}" \
|
||||
&& [[ -S "${ROBOT_RECEIVER_LOCAL_SOCKET_PATH}" ]] \
|
||||
&& status_file_fresh "${ROS_STATUS_FILE}" "${max_age_sec}" \
|
||||
&& ros_receiver_status_fresh "${ROS_STATUS_FILE}" "${max_age_sec}"
|
||||
}
|
||||
|
||||
write_watchdog_status() {
|
||||
local fault_reason="$1"
|
||||
local recovery_state="$2"
|
||||
local network_ok="$3"
|
||||
local camera_ok="$4"
|
||||
local ros_ok="$5"
|
||||
local bside_ok="$6"
|
||||
local gps_ok="$7"
|
||||
local gps_device_present="$8"
|
||||
local tmp_file
|
||||
|
||||
tmp_file="${WATCHDOG_STATUS_FILE}.tmp.$$"
|
||||
cat > "${tmp_file}" <<EOF
|
||||
{
|
||||
"updated_at_epoch_ms": $(now_epoch_ms),
|
||||
"fault_reason": "${fault_reason}",
|
||||
"recovery_state": "${recovery_state}",
|
||||
"network_ok": ${network_ok},
|
||||
"camera_ok": ${camera_ok},
|
||||
"ros_ok": ${ros_ok},
|
||||
"bside_ok": ${bside_ok},
|
||||
"gps_ok": ${gps_ok},
|
||||
"gps_device_present": ${gps_device_present},
|
||||
"network_fail_count": ${NETWORK_FAIL_COUNT},
|
||||
"targeted_restart_count": $(targeted_restart_total),
|
||||
"full_restart_count": ${FULL_RESTART_WINDOW_COUNT},
|
||||
"last_action": "${LAST_ACTION}",
|
||||
"last_action_epoch_ms": ${LAST_ACTION_EPOCH_MS}
|
||||
}
|
||||
EOF
|
||||
mv -f "${tmp_file}" "${WATCHDOG_STATUS_FILE}"
|
||||
}
|
||||
|
||||
watchdog_emit_json() {
|
||||
local record_type="$1"
|
||||
local action="$2"
|
||||
local fault_reason="$3"
|
||||
local recovery_state="$4"
|
||||
local detail="$5"
|
||||
local incident_id="${6:-}"
|
||||
local network_ok="${7:-1}"
|
||||
local camera_ok="${8:-1}"
|
||||
local ros_ok="${9:-1}"
|
||||
local bside_ok="${10:-1}"
|
||||
local gps_ok="${11:-1}"
|
||||
local gps_device_present="${12:-1}"
|
||||
|
||||
python3 - "${record_type}" "${action}" "${fault_reason}" "${recovery_state}" "${detail}" "${incident_id}" "${network_ok}" "${camera_ok}" "${ros_ok}" "${bside_ok}" "${gps_ok}" "${gps_device_present}" "${LAST_ACTION}" "${LAST_ACTION_EPOCH_MS}" "${NETWORK_FAIL_COUNT}" "$(targeted_restart_total)" "${FULL_RESTART_WINDOW_COUNT}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
record_type, action, fault_reason, recovery_state, detail, incident_id, network_ok, camera_ok, ros_ok, bside_ok, gps_ok, gps_device_present, last_action, last_action_epoch_ms, network_fail_count, targeted_restart_count, full_restart_count = sys.argv[1:18]
|
||||
payload = {
|
||||
"ts_unix_ms": time.time_ns() // 1_000_000,
|
||||
"record_type": record_type,
|
||||
"action": action,
|
||||
"fault_reason": fault_reason,
|
||||
"recovery_state": recovery_state,
|
||||
"detail": detail,
|
||||
"incident_id": incident_id or None,
|
||||
"network_ok": network_ok == "1",
|
||||
"camera_ok": camera_ok == "1",
|
||||
"ros_ok": ros_ok == "1",
|
||||
"bside_ok": bside_ok == "1",
|
||||
"gps_ok": gps_ok == "1",
|
||||
"gps_device_present": gps_device_present == "1",
|
||||
"network_fail_count": int(network_fail_count),
|
||||
"targeted_restart_count": int(targeted_restart_count),
|
||||
"full_restart_count": int(full_restart_count),
|
||||
"last_action": last_action,
|
||||
"last_action_epoch_ms": int(last_action_epoch_ms or 0),
|
||||
}
|
||||
print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
watchdog_append_event() {
|
||||
local line=""
|
||||
|
||||
[[ -n "${WATCHDOG_EVENT_LOG}" ]] || return 0
|
||||
if ! line="$(watchdog_emit_json "$@" 2>&1)"; then
|
||||
if (( WATCHDOG_EVENT_LOG_FAILURE_REPORTED == 0 )); then
|
||||
blitz_log "${STEP}" "watchdog-event-log" "failure" "path=${WATCHDOG_EVENT_LOG} detail=${line}" 0 || true
|
||||
WATCHDOG_EVENT_LOG_FAILURE_REPORTED=1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
if ! blitz_jsonl_append_line "${WATCHDOG_EVENT_LOG}" "${line}"; then
|
||||
if (( WATCHDOG_EVENT_LOG_FAILURE_REPORTED == 0 )); then
|
||||
blitz_log "${STEP}" "watchdog-event-log" "failure" "path=${WATCHDOG_EVENT_LOG} detail=append-failed" 0 || true
|
||||
WATCHDOG_EVENT_LOG_FAILURE_REPORTED=1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
WATCHDOG_EVENT_LOG_FAILURE_REPORTED=0
|
||||
}
|
||||
|
||||
watchdog_append_sample() {
|
||||
local line=""
|
||||
|
||||
[[ -n "${WATCHDOG_SAMPLE_LOG}" ]] || return 0
|
||||
if ! line="$(watchdog_emit_json "$@" 2>&1)"; then
|
||||
if (( WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED == 0 )); then
|
||||
blitz_log "${STEP}" "watchdog-sample-log" "failure" "path=${WATCHDOG_SAMPLE_LOG} detail=${line}" 0 || true
|
||||
WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED=1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
if ! blitz_jsonl_append_line "${WATCHDOG_SAMPLE_LOG}" "${line}"; then
|
||||
if (( WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED == 0 )); then
|
||||
blitz_log "${STEP}" "watchdog-sample-log" "failure" "path=${WATCHDOG_SAMPLE_LOG} detail=append-failed" 0 || true
|
||||
WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED=1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
WATCHDOG_SAMPLE_LOG_FAILURE_REPORTED=0
|
||||
}
|
||||
|
||||
watchdog_record_state_transition() {
|
||||
local fault_reason="$1"
|
||||
local recovery_state="$2"
|
||||
|
||||
if [[ "${fault_reason}" == "${LAST_REPORTED_FAULT_REASON}" && "${recovery_state}" == "${LAST_REPORTED_RECOVERY_STATE}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
watchdog_append_event "event" "state-transition" "${fault_reason}" "${recovery_state}" "" ""
|
||||
LAST_REPORTED_FAULT_REASON="${fault_reason}"
|
||||
LAST_REPORTED_RECOVERY_STATE="${recovery_state}"
|
||||
}
|
||||
|
||||
watchdog_launch_incident() {
|
||||
local reason="$1"
|
||||
local unit_name="$2"
|
||||
|
||||
blitz_launch_incident_capture \
|
||||
--source watchdog \
|
||||
--reason "${reason}" \
|
||||
--unit "${unit_name}" \
|
||||
--result failure \
|
||||
--exit-status 1 2>/dev/null || true
|
||||
}
|
||||
|
||||
set_last_action() {
|
||||
LAST_ACTION="$1"
|
||||
LAST_ACTION_EPOCH_MS="$(now_epoch_ms)"
|
||||
}
|
||||
|
||||
targeted_restart_total() {
|
||||
local total=0
|
||||
local key
|
||||
|
||||
for key in "${!TARGETED_RESTART_WINDOW_COUNT[@]}"; do
|
||||
total=$(( total + TARGETED_RESTART_WINDOW_COUNT["${key}"] ))
|
||||
done
|
||||
printf '%s\n' "${total}"
|
||||
}
|
||||
|
||||
register_targeted_restart() {
|
||||
local fault_key="$1"
|
||||
local now_sec
|
||||
local window_start
|
||||
local count
|
||||
|
||||
now_sec="$(now_epoch_sec)"
|
||||
window_start="${TARGETED_RESTART_WINDOW_START["${fault_key}"]:-0}"
|
||||
count="${TARGETED_RESTART_WINDOW_COUNT["${fault_key}"]:-0}"
|
||||
if (( window_start == 0 || now_sec - window_start > 60 )); then
|
||||
window_start="${now_sec}"
|
||||
count=1
|
||||
else
|
||||
count=$(( count + 1 ))
|
||||
fi
|
||||
TARGETED_RESTART_WINDOW_START["${fault_key}"]="${window_start}"
|
||||
TARGETED_RESTART_WINDOW_COUNT["${fault_key}"]="${count}"
|
||||
(( count >= 2 ))
|
||||
}
|
||||
|
||||
record_full_restart() {
|
||||
local now_sec
|
||||
|
||||
now_sec="$(now_epoch_sec)"
|
||||
if (( FULL_RESTART_WINDOW_START == 0 || now_sec - FULL_RESTART_WINDOW_START > 600 )); then
|
||||
FULL_RESTART_WINDOW_START="${now_sec}"
|
||||
FULL_RESTART_WINDOW_COUNT=1
|
||||
else
|
||||
FULL_RESTART_WINDOW_COUNT=$(( FULL_RESTART_WINDOW_COUNT + 1 ))
|
||||
fi
|
||||
if (( FULL_RESTART_WINDOW_COUNT >= 3 )); then
|
||||
BACKOFF_UNTIL=$(( now_sec + 60 ))
|
||||
watchdog_append_event "event" "backoff-enter" "backoff" "backoff" "full_restart_count=${FULL_RESTART_WINDOW_COUNT}" ""
|
||||
fi
|
||||
}
|
||||
|
||||
restart_bside_targeted() {
|
||||
local fault_key="$1"
|
||||
local reason="$2"
|
||||
local rc
|
||||
local incident_id=""
|
||||
|
||||
if register_targeted_restart "${fault_key}"; then
|
||||
blitz_log "${STEP}" "escalate-full-restart" "start" "reason=${reason}" 0
|
||||
watchdog_append_event "event" "escalate-full-restart" "${reason}-escalated" "recovering" "fault_key=${fault_key}" ""
|
||||
full_restart_stack "${reason}-escalated"
|
||||
return 0
|
||||
fi
|
||||
|
||||
incident_id="$(watchdog_launch_incident "${reason}" "${B_SIDE_SERVICE}")"
|
||||
set_last_action "restart-bside"
|
||||
RECOVERY_ACTION_TAKEN=1
|
||||
blitz_log "${STEP}" "restart-bside" "start" "reason=${reason}" 0
|
||||
watchdog_append_event "event" "restart-bside-start" "${reason}" "recovering" "fault_key=${fault_key}" "${incident_id}"
|
||||
if systemctl restart "${B_SIDE_SERVICE}"; then
|
||||
blitz_log "${STEP}" "restart-bside" "success" "reason=${reason}" 0
|
||||
watchdog_append_event "event" "restart-bside-success" "${reason}" "recovering" "fault_key=${fault_key}" "${incident_id}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "restart-bside" "failure" "reason=${reason}" "${rc}"
|
||||
watchdog_append_event "event" "restart-bside-failure" "${reason}" "recovering" "fault_key=${fault_key} rc=${rc}" "${incident_id}"
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
full_restart_stack() {
|
||||
local reason="$1"
|
||||
local rc
|
||||
local incident_id=""
|
||||
|
||||
incident_id="$(watchdog_launch_incident "${reason}" "blitz-robot.target")"
|
||||
set_last_action "full-restart"
|
||||
RECOVERY_ACTION_TAKEN=1
|
||||
recovery_state="recovering"
|
||||
fault_reason="${reason}"
|
||||
|
||||
blitz_log "${STEP}" "full-restart-stop-bside" "start" "reason=${reason}" 0
|
||||
watchdog_append_event "event" "full-restart-start" "${reason}" "recovering" "" "${incident_id}"
|
||||
systemctl stop "${B_SIDE_SERVICE}" || true
|
||||
|
||||
if systemctl restart "${ROS_SERVICE}"; then
|
||||
blitz_log "${STEP}" "full-restart-restart-ros" "success" "reason=${reason}" 0
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "full-restart-restart-ros" "failure" "reason=${reason}" "${rc}"
|
||||
record_full_restart
|
||||
return "${rc}"
|
||||
fi
|
||||
|
||||
if bash "${BOOT_SCRIPT_DIR}/wait-for-unix-socket.sh" --step "${STEP}" --timeout "${BLITZ_ROS_SOCKET_WAIT_SEC}"; then
|
||||
:
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "full-restart-wait-socket" "failure" "reason=${reason}" "${rc}"
|
||||
record_full_restart
|
||||
return "${rc}"
|
||||
fi
|
||||
|
||||
if systemctl start "${B_SIDE_SERVICE}"; then
|
||||
blitz_log "${STEP}" "full-restart-start-bside" "success" "reason=${reason}" 0
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "full-restart-start-bside" "failure" "reason=${reason}" "${rc}"
|
||||
watchdog_append_event "event" "full-restart-failure" "${reason}" "recovering" "stage=start-bside rc=${rc}" "${incident_id}"
|
||||
record_full_restart
|
||||
return "${rc}"
|
||||
fi
|
||||
watchdog_append_event "event" "full-restart-success" "${reason}" "recovering" "" "${incident_id}"
|
||||
record_full_restart
|
||||
}
|
||||
|
||||
network_fault_injected() {
|
||||
[[ "${BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION}" == "1" && -f "${NETWORK_FAULT_FILE}" ]]
|
||||
}
|
||||
|
||||
resolve_network_interface() {
|
||||
NETWORK_LAST_INTERFACE="$(blitz_resolve_5g_interface || true)"
|
||||
if [[ -n "${NETWORK_LAST_INTERFACE}" ]]; then
|
||||
NETWORK_ROUTE_INTERFACE_LAST_KNOWN="${NETWORK_LAST_INTERFACE}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
network_route_targets() {
|
||||
local target
|
||||
|
||||
if [[ -n "${BLITZ_TIME_SERVER_IP:-}" ]]; then
|
||||
printf '%s\n' "${BLITZ_TIME_SERVER_IP}"
|
||||
fi
|
||||
for target in ${BLITZ_5G_ROUTE_TARGETS//,/ }; do
|
||||
if [[ -n "${target}" && "${target}" != "${BLITZ_TIME_SERVER_IP:-}" ]]; then
|
||||
printf '%s\n' "${target}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
log_target_route_paths() {
|
||||
local action="$1"
|
||||
local target
|
||||
local route_output
|
||||
|
||||
while IFS= read -r target; do
|
||||
[[ -n "${target}" ]] || continue
|
||||
route_output="$(ip route get "${target}" 2>&1 | head -n 1 || true)"
|
||||
if [[ -z "${route_output}" ]]; then
|
||||
route_output="unresolved"
|
||||
fi
|
||||
blitz_log "${STEP}" "route-path" "info" "action=${action} target=${target} route=${route_output}" 0
|
||||
done < <(network_route_targets)
|
||||
}
|
||||
|
||||
route_output_uses_interface() {
|
||||
local route_output="$1"
|
||||
local interface_name="$2"
|
||||
|
||||
[[ -n "${interface_name}" ]] || return 1
|
||||
[[ "${route_output}" == *" dev ${interface_name} "* || "${route_output}" == *" dev ${interface_name}" ]]
|
||||
}
|
||||
|
||||
route_output_uses_gateway() {
|
||||
local route_output="$1"
|
||||
local gateway="$2"
|
||||
|
||||
[[ -n "${gateway}" ]] || return 1
|
||||
[[ "${route_output}" == *"via ${gateway}"* ]]
|
||||
}
|
||||
|
||||
route_is_desired_target_route() {
|
||||
local route_output="$1"
|
||||
local interface_name="$2"
|
||||
local gateway="$3"
|
||||
|
||||
route_output_uses_interface "${route_output}" "${interface_name}" \
|
||||
&& route_output_uses_gateway "${route_output}" "${gateway}"
|
||||
}
|
||||
|
||||
route_is_managed_5g_route() {
|
||||
local route_output="$1"
|
||||
local interface_name="${2:-}"
|
||||
local gateway="${3:-}"
|
||||
|
||||
if route_output_uses_interface "${route_output}" "${interface_name}"; then
|
||||
return 0
|
||||
fi
|
||||
if route_output_uses_gateway "${route_output}" "${gateway}"; then
|
||||
return 0
|
||||
fi
|
||||
if route_output_uses_gateway "${route_output}" "${BLITZ_5G_GATEWAY:-}"; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
resolve_route_cleanup_interface() {
|
||||
local interface_name=""
|
||||
local info_json="${BLITZ_5G_INFO_JSON:-}"
|
||||
|
||||
if [[ -n "${NETWORK_LAST_INTERFACE}" ]]; then
|
||||
printf '%s\n' "${NETWORK_LAST_INTERFACE}"
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${NETWORK_ROUTE_INTERFACE_LAST_KNOWN}" ]]; then
|
||||
printf '%s\n' "${NETWORK_ROUTE_INTERFACE_LAST_KNOWN}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
interface_name="$(blitz_read_5g_info_interface "${info_json}" || true)"
|
||||
if [[ -n "${interface_name}" ]]; then
|
||||
printf '%s\n' "${interface_name}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
resolve_network_gateway() {
|
||||
local interface_name="$1"
|
||||
local default_route
|
||||
local gateway=""
|
||||
local tokens=()
|
||||
local index
|
||||
|
||||
default_route="$(ip -o route show default dev "${interface_name}" 2>/dev/null | head -n 1 || true)"
|
||||
if [[ -n "${default_route}" ]]; then
|
||||
read -r -a tokens <<< "${default_route}"
|
||||
for (( index=0; index<${#tokens[@]}-1; index++ )); do
|
||||
if [[ "${tokens[index]}" == "via" ]]; then
|
||||
gateway="${tokens[index + 1]}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -n "${gateway}" ]]; then
|
||||
printf '%s\n' "${gateway}"
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${BLITZ_5G_GATEWAY:-}" ]]; then
|
||||
printf '%s\n' "${BLITZ_5G_GATEWAY}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
sync_target_routes_to_5g() {
|
||||
local interface_name="$1"
|
||||
local gateway="${2:-}"
|
||||
local route_output=""
|
||||
local updated=0
|
||||
local target
|
||||
local rc
|
||||
|
||||
if [[ -z "${interface_name}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "${gateway}" ]]; then
|
||||
gateway="$(resolve_network_gateway "${interface_name}" || true)"
|
||||
fi
|
||||
if [[ -z "${gateway}" ]]; then
|
||||
blitz_log "${STEP}" "route-sync-gateway" "failure" "interface=${interface_name}" 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS= read -r target; do
|
||||
[[ -n "${target}" ]] || continue
|
||||
route_output="$(ip route show "${target}/32" 2>/dev/null | head -n 1 || true)"
|
||||
if [[ -n "${route_output}" ]] && route_is_desired_target_route "${route_output}" "${interface_name}" "${gateway}"; then
|
||||
continue
|
||||
fi
|
||||
if ip route replace "${target}/32" via "${gateway}" dev "${interface_name}"; then
|
||||
updated=1
|
||||
blitz_log "${STEP}" "route-sync-target" "success" "target=${target} interface=${interface_name} gateway=${gateway}" 0
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "route-sync-target" "failure" "target=${target} interface=${interface_name} gateway=${gateway}" "${rc}"
|
||||
return "${rc}"
|
||||
fi
|
||||
done < <(network_route_targets)
|
||||
|
||||
if (( updated == 1 )); then
|
||||
NETWORK_ROUTE_INTERFACE_LAST_KNOWN="${interface_name}"
|
||||
log_target_route_paths "sync-to-5g"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
clear_target_routes_from_5g() {
|
||||
local interface_name="${1:-}"
|
||||
local gateway="${2:-}"
|
||||
local route_output=""
|
||||
local target
|
||||
local removed_any=0
|
||||
local rc
|
||||
|
||||
if [[ -z "${interface_name}" ]]; then
|
||||
interface_name="$(resolve_route_cleanup_interface || true)"
|
||||
fi
|
||||
if [[ -z "${gateway}" && -n "${interface_name}" ]]; then
|
||||
gateway="$(resolve_network_gateway "${interface_name}" || true)"
|
||||
fi
|
||||
if [[ -z "${gateway}" ]]; then
|
||||
gateway="${BLITZ_5G_GATEWAY:-}"
|
||||
fi
|
||||
|
||||
while IFS= read -r target; do
|
||||
[[ -n "${target}" ]] || continue
|
||||
route_output="$(ip route show "${target}/32" 2>/dev/null | head -n 1 || true)"
|
||||
if [[ -z "${route_output}" ]] || ! route_is_managed_5g_route "${route_output}" "${interface_name}" "${gateway}"; then
|
||||
continue
|
||||
fi
|
||||
if ip route del "${target}/32"; then
|
||||
removed_any=1
|
||||
blitz_log "${STEP}" "route-clear-target" "success" "target=${target} interface=${interface_name:-unknown} gateway=${gateway:-unknown}" 0
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "route-clear-target" "failure" "target=${target} interface=${interface_name:-unknown} gateway=${gateway:-unknown}" "${rc}"
|
||||
return "${rc}"
|
||||
fi
|
||||
done < <(network_route_targets)
|
||||
|
||||
if (( removed_any == 1 )); then
|
||||
blitz_log "${STEP}" "route-clear" "success" "interface=${interface_name:-unknown} gateway=${gateway:-unknown}" 0
|
||||
log_target_route_paths "clear-from-5g"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
repair_network_routes() {
|
||||
local interface_name="$1"
|
||||
local gateway=""
|
||||
local route_output
|
||||
|
||||
if [[ -z "${interface_name}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
gateway="$(resolve_network_gateway "${interface_name}" || true)"
|
||||
if [[ -z "${gateway}" ]]; then
|
||||
blitz_log "${STEP}" "route-repair-gateway" "failure" "interface=${interface_name}" 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! sync_target_routes_to_5g "${interface_name}" "${gateway}"; then
|
||||
clear_target_routes_from_5g "${interface_name}" "${gateway}" || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
route_output="$(blitz_route_ready "${BLITZ_TIME_SERVER_IP}" "${interface_name}" || true)"
|
||||
if [[ -z "${route_output}" ]]; then
|
||||
clear_target_routes_from_5g "${interface_name}" "${gateway}" || true
|
||||
blitz_log "${STEP}" "route-repair-postcheck" "failure" "interface=${interface_name} gateway=${gateway}" 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! ping -I "${interface_name}" -c 1 -W 2 "${BLITZ_TIME_SERVER_IP}" >/dev/null 2>&1; then
|
||||
clear_target_routes_from_5g "${interface_name}" "${gateway}" || true
|
||||
blitz_log "${STEP}" "route-repair-probe" "failure" "interface=${interface_name} target=${BLITZ_TIME_SERVER_IP}" 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
blitz_log "${STEP}" "route-repair-postcheck" "success" "interface=${interface_name} gateway=${gateway} route=${route_output}" 0
|
||||
return 0
|
||||
}
|
||||
|
||||
network_is_healthy() {
|
||||
local route_output
|
||||
|
||||
NETWORK_LAST_INTERFACE=""
|
||||
if network_fault_injected; then
|
||||
return 1
|
||||
fi
|
||||
if ! resolve_network_interface; then
|
||||
return 1
|
||||
fi
|
||||
route_output="$(blitz_route_ready "${BLITZ_TIME_SERVER_IP}" "${NETWORK_LAST_INTERFACE}" || true)"
|
||||
if [[ -z "${route_output}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
ping -I "${NETWORK_LAST_INTERFACE}" -c 1 -W 2 "${BLITZ_TIME_SERVER_IP}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
fallback_network_is_healthy() {
|
||||
local route_output
|
||||
|
||||
if [[ -z "${BLITZ_TIME_SERVER_IP:-}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
route_output="$(blitz_route_ready "${BLITZ_TIME_SERVER_IP}" || true)"
|
||||
if [[ -z "${route_output}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
ping -c 1 -W 2 "${BLITZ_TIME_SERVER_IP}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
wait_for_network_recovery() {
|
||||
local timeout_sec="$1"
|
||||
local waited=0
|
||||
|
||||
while (( waited < timeout_sec )); do
|
||||
if network_is_healthy; then
|
||||
blitz_log "${STEP}" "network-postcheck" "success" "interface=${NETWORK_LAST_INTERFACE} waited_sec=${waited}" 0
|
||||
return 0
|
||||
fi
|
||||
if (( waited == 0 || waited % 5 == 0 )); then
|
||||
blitz_log "${STEP}" "network-postcheck" "waiting" "interface=${NETWORK_LAST_INTERFACE:-unresolved} waited_sec=${waited}" 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$(( waited + 1 ))
|
||||
done
|
||||
|
||||
blitz_log "${STEP}" "network-postcheck" "failure" "interface=${NETWORK_LAST_INTERFACE:-unresolved} timeout_sec=${timeout_sec}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
perform_network_recovery() {
|
||||
local rc=0
|
||||
local incident_id=""
|
||||
|
||||
if resolve_network_interface && repair_network_routes "${NETWORK_LAST_INTERFACE}"; then
|
||||
set_last_action "route-repair"
|
||||
RECOVERY_ACTION_TAKEN=1
|
||||
NETWORK_COOLDOWN_UNTIL=$(( $(now_epoch_sec) + BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC ))
|
||||
NETWORK_FAIL_COUNT=0
|
||||
blitz_log "${STEP}" "network-recovery" "success" "mode=route-repair interface=${NETWORK_LAST_INTERFACE}" 0
|
||||
watchdog_append_event "event" "route-repair-success" "network_or_robot_unreachable" "recovering" "interface=${NETWORK_LAST_INTERFACE}" ""
|
||||
return 0
|
||||
fi
|
||||
|
||||
incident_id="$(watchdog_launch_incident "network-recovery" "blitz-5g-dial.service")"
|
||||
set_last_action "network-recovery"
|
||||
RECOVERY_ACTION_TAKEN=1
|
||||
blitz_log "${STEP}" "network-recovery" "start" "fail_count=${NETWORK_FAIL_COUNT}" 0
|
||||
watchdog_append_event "event" "network-recovery-start" "network_or_robot_unreachable" "recovering" "fail_count=${NETWORK_FAIL_COUNT}" "${incident_id}"
|
||||
systemctl stop "${B_SIDE_SERVICE}" || true
|
||||
|
||||
if bash "${BOOT_SCRIPT_DIR}/5g-dial.sh"; then
|
||||
:
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "network-redial" "failure" "fail_count=${NETWORK_FAIL_COUNT} script=${BOOT_SCRIPT_DIR}/5g-dial.sh" "${rc}"
|
||||
watchdog_append_event "event" "network-recovery-failure" "network_or_robot_unreachable" "recovering" "stage=redial rc=${rc}" "${incident_id}"
|
||||
return "${rc}"
|
||||
fi
|
||||
|
||||
if wait_for_network_recovery "${BLITZ_5G_ROUTE_WAIT_SEC}"; then
|
||||
:
|
||||
else
|
||||
rc=$?
|
||||
blitz_log "${STEP}" "network-recovery" "failure" "fail_count=${NETWORK_FAIL_COUNT} interface=${NETWORK_LAST_INTERFACE:-unresolved}" "${rc}"
|
||||
watchdog_append_event "event" "network-recovery-failure" "network_or_robot_unreachable" "recovering" "stage=postcheck rc=${rc}" "${incident_id}"
|
||||
return "${rc}"
|
||||
fi
|
||||
|
||||
NETWORK_COOLDOWN_UNTIL=$(( $(now_epoch_sec) + BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC ))
|
||||
NETWORK_FAIL_COUNT=0
|
||||
watchdog_append_event "event" "network-recovery-success" "network_or_robot_unreachable" "recovering" "interface=${NETWORK_LAST_INTERFACE:-unresolved}" "${incident_id}"
|
||||
if ros_receiver_healthy "${BLITZ_HEALTH_STALE_SEC}"; then
|
||||
restart_bside_targeted "network" "network-recovered"
|
||||
return 0
|
||||
fi
|
||||
full_restart_stack "network-recovered-ros-unhealthy"
|
||||
return 0
|
||||
}
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_require_command systemctl "${STEP}"
|
||||
blitz_require_command stat "${STEP}"
|
||||
blitz_require_command ping "${STEP}"
|
||||
blitz_require_command python3 "${STEP}"
|
||||
blitz_prepare_runtime_dir
|
||||
blitz_require_run_context
|
||||
|
||||
B_SIDE_STATUS_FILE="${BLITZ_RUNTIME_DIR}/b-side-omnid.status.json"
|
||||
ROS_STATUS_FILE="${BLITZ_RUNTIME_DIR}/ros-receiver.status.json"
|
||||
WATCHDOG_STATUS_FILE="${BLITZ_RUNTIME_DIR}/watchdog.status.json"
|
||||
NETWORK_FAULT_FILE="${BLITZ_RUNTIME_DIR}/fault-injection-network-down"
|
||||
WATCHDOG_EVENT_LOG="${BLITZ_RUN_DIR}/watchdog-events.jsonl"
|
||||
WATCHDOG_SAMPLE_LOG="${BLITZ_RUN_DIR}/watchdog-samples.jsonl"
|
||||
|
||||
while true; do
|
||||
fault_reason="none"
|
||||
recovery_state="ok"
|
||||
network_ok=1
|
||||
camera_ok=1
|
||||
ros_ok=1
|
||||
bside_ok=1
|
||||
gps_ok=1
|
||||
gps_device_present=1
|
||||
RECOVERY_ACTION_TAKEN=0
|
||||
now_sec="$(now_epoch_sec)"
|
||||
|
||||
if gps_monitor_enabled; then
|
||||
gps_device_present="${GPS_DEVICE_PRESENT_STATE}"
|
||||
if (( GPS_DEVICE_PRESENT_STATE == 0 || GPS_STACK_ACTIVE_STATE == 0 )); then
|
||||
gps_ok=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( BACKOFF_UNTIL > now_sec )); then
|
||||
fault_reason="backoff"
|
||||
recovery_state="backoff"
|
||||
watchdog_record_state_transition "${fault_reason}" "${recovery_state}"
|
||||
write_watchdog_status "${fault_reason}" "${recovery_state}" 0 0 0 0 "${gps_ok}" "${gps_device_present}"
|
||||
watchdog_append_sample "sample" "loop" "${fault_reason}" "${recovery_state}" "" "" 0 0 0 0 "${gps_ok}" "${gps_device_present}"
|
||||
sleep "${BLITZ_WATCHDOG_INTERVAL_SEC}"
|
||||
continue
|
||||
fi
|
||||
|
||||
if (( NETWORK_COOLDOWN_UNTIL > now_sec )); then
|
||||
recovery_state="recovering"
|
||||
elif ! network_is_healthy; then
|
||||
clear_target_routes_from_5g || true
|
||||
if fallback_network_is_healthy; then
|
||||
NETWORK_FAIL_COUNT=0
|
||||
fault_reason="network_fallback_active"
|
||||
recovery_state="degraded"
|
||||
blitz_log "${STEP}" "network-check" "fallback" "interface=${NETWORK_LAST_INTERFACE:-unresolved} target=${BLITZ_TIME_SERVER_IP}" 0
|
||||
if (( NETWORK_PRIMARY_LAST_RETRY_SEC == 0 || now_sec - NETWORK_PRIMARY_LAST_RETRY_SEC >= 10 )); then
|
||||
NETWORK_PRIMARY_LAST_RETRY_SEC="${now_sec}"
|
||||
if resolve_network_interface && repair_network_routes "${NETWORK_LAST_INTERFACE}"; then
|
||||
NETWORK_PRIMARY_LAST_RETRY_SEC=0
|
||||
fault_reason="none"
|
||||
recovery_state="ok"
|
||||
blitz_log "${STEP}" "network-check" "primary-restored" "interface=${NETWORK_LAST_INTERFACE} target=${BLITZ_TIME_SERVER_IP}" 0
|
||||
log_target_route_paths "primary-restored"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
network_ok=0
|
||||
NETWORK_FAIL_COUNT=$(( NETWORK_FAIL_COUNT + 1 ))
|
||||
fault_reason="network_or_robot_unreachable"
|
||||
recovery_state="recovering"
|
||||
blitz_log "${STEP}" "network-check" "failure" "count=${NETWORK_FAIL_COUNT} interface=${NETWORK_LAST_INTERFACE:-unresolved}" 1
|
||||
if (( NETWORK_FAIL_COUNT >= BLITZ_NETWORK_FAIL_THRESHOLD )); then
|
||||
perform_network_recovery || true
|
||||
fi
|
||||
fi
|
||||
else
|
||||
NETWORK_PRIMARY_LAST_RETRY_SEC=0
|
||||
NETWORK_FAIL_COUNT=0
|
||||
sync_target_routes_to_5g "${NETWORK_LAST_INTERFACE}" || true
|
||||
fi
|
||||
|
||||
if check_gps_health "${now_sec}"; then
|
||||
gps_ok=1
|
||||
else
|
||||
gps_ok=0
|
||||
gps_device_present="${GPS_DEVICE_PRESENT_STATE}"
|
||||
if [[ "${fault_reason}" == "none" ]]; then
|
||||
if (( GPS_DEVICE_PRESENT_STATE == 0 )); then
|
||||
fault_reason="gps_device_missing"
|
||||
else
|
||||
fault_reason="gps_reconnect_failed"
|
||||
fi
|
||||
recovery_state="degraded"
|
||||
fi
|
||||
fi
|
||||
gps_device_present="${GPS_DEVICE_PRESENT_STATE}"
|
||||
|
||||
if [[ ! -e "${OMNI_CAMERA_DEVICE}" ]]; then
|
||||
camera_ok=0
|
||||
fault_reason="camera_missing"
|
||||
recovery_state="degraded"
|
||||
CAMERA_MISSING_PREV=1
|
||||
CAMERA_RECOVERY_STABLE_COUNT=0
|
||||
elif (( RECOVERY_ACTION_TAKEN == 0 && CAMERA_MISSING_PREV == 1 )); then
|
||||
CAMERA_RECOVERY_STABLE_COUNT=$(( CAMERA_RECOVERY_STABLE_COUNT + 1 ))
|
||||
recovery_state="recovering"
|
||||
fault_reason="camera_recovered"
|
||||
if (( CAMERA_RECOVERY_STABLE_COUNT >= 2 )); then
|
||||
restart_bside_targeted "camera" "camera-reappeared" || true
|
||||
CAMERA_MISSING_PREV=0
|
||||
CAMERA_RECOVERY_STABLE_COUNT=0
|
||||
fi
|
||||
else
|
||||
CAMERA_RECOVERY_STABLE_COUNT=0
|
||||
fi
|
||||
|
||||
if (( RECOVERY_ACTION_TAKEN == 0 )) && { ! service_is_active "${B_SIDE_SERVICE}" || ! status_file_fresh "${B_SIDE_STATUS_FILE}" "${BLITZ_HEALTH_STALE_SEC}"; }; then
|
||||
bside_ok=0
|
||||
fault_reason="bside_status_stale"
|
||||
recovery_state="recovering"
|
||||
restart_bside_targeted "bside" "bside-unhealthy" || true
|
||||
fi
|
||||
|
||||
if (( RECOVERY_ACTION_TAKEN == 0 )) && ! ros_receiver_healthy "${BLITZ_HEALTH_STALE_SEC}"; then
|
||||
ros_ok=0
|
||||
fault_reason="ros_receiver_unhealthy"
|
||||
recovery_state="recovering"
|
||||
full_restart_stack "ros-unhealthy" || true
|
||||
fi
|
||||
|
||||
watchdog_record_state_transition "${fault_reason}" "${recovery_state}"
|
||||
write_watchdog_status "${fault_reason}" "${recovery_state}" "${network_ok}" "${camera_ok}" "${ros_ok}" "${bside_ok}" "${gps_ok}" "${gps_device_present}"
|
||||
watchdog_append_sample "sample" "loop" "${fault_reason}" "${recovery_state}" "" "" "${network_ok}" "${camera_ok}" "${ros_ok}" "${bside_ok}" "${gps_ok}" "${gps_device_present}"
|
||||
sleep "${BLITZ_WATCHDOG_INTERVAL_SEC}"
|
||||
done
|
||||
15
robot/v4l2/OmniSocketGo_robot/scripts/boot/boot-gate.sh
Normal file
15
robot/v4l2/OmniSocketGo_robot/scripts/boot/boot-gate.sh
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="boot-gate"
|
||||
|
||||
blitz_load_boot_env
|
||||
|
||||
blitz_log "${STEP}" "start" "start" "delay_sec=${BLITZ_BOOT_DELAY_SEC}" 0
|
||||
blitz_log "${STEP}" "delay" "start" "sleep ${BLITZ_BOOT_DELAY_SEC}s before starting Blitz services" 0
|
||||
sleep "${BLITZ_BOOT_DELAY_SEC}"
|
||||
blitz_log "${STEP}" "delay" "success" "boot gate released after ${BLITZ_BOOT_DELAY_SEC}s" 0
|
||||
661
robot/v4l2/OmniSocketGo_robot/scripts/boot/common.sh
Normal file
661
robot/v4l2/OmniSocketGo_robot/scripts/boot/common.sh
Normal file
@@ -0,0 +1,661 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BOOT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEV_SCRIPT_DIR="$(cd "${BOOT_SCRIPT_DIR}/../dev" && pwd)"
|
||||
|
||||
source_with_nounset_off() {
|
||||
set +u
|
||||
# shellcheck disable=SC1090
|
||||
source "$1"
|
||||
set -u
|
||||
}
|
||||
|
||||
blitz_host_from_addr() {
|
||||
local value="${1:-}"
|
||||
|
||||
if [[ -z "${value}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${value}" == \[*\]:* ]]; then
|
||||
value="${value#\[}"
|
||||
printf '%s\n' "${value%%]:*}"
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${value%%:*}"
|
||||
}
|
||||
|
||||
blitz_load_boot_env() {
|
||||
local env_file
|
||||
local default_time_server
|
||||
local dev_run_root
|
||||
local dev_runtime_dir
|
||||
|
||||
if [[ "${BLITZ_BOOT_ENV_LOADED:-0}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
export BLITZ_BOOT_LOADING_ENV="1"
|
||||
# shellcheck disable=SC1091
|
||||
source "${DEV_SCRIPT_DIR}/load-env.sh"
|
||||
unset BLITZ_BOOT_LOADING_ENV
|
||||
|
||||
for env_file in \
|
||||
"${BOOT_SCRIPT_DIR}/robot-boot.env" \
|
||||
"${BOOT_SCRIPT_DIR}/robot-boot.env.local"
|
||||
do
|
||||
if [[ -f "${env_file}" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${env_file}"
|
||||
set +a
|
||||
fi
|
||||
done
|
||||
|
||||
if declare -F normalize_loaded_env_vars >/dev/null 2>&1; then
|
||||
normalize_loaded_env_vars
|
||||
fi
|
||||
|
||||
dev_run_root="${OMNISOCKETGO_ROOT}/logs"
|
||||
dev_runtime_dir="${dev_run_root}/runtime"
|
||||
|
||||
if [[ -z "${BLITZ_RUN_ROOT:-}" || "${BLITZ_RUN_ROOT}" == "${dev_run_root}" ]]; then
|
||||
export BLITZ_RUN_ROOT="/var/log/blitz-robot"
|
||||
fi
|
||||
if [[ -z "${BLITZ_RUNTIME_DIR:-}" || "${BLITZ_RUNTIME_DIR}" == "${dev_runtime_dir}" ]]; then
|
||||
export BLITZ_RUNTIME_DIR="/run/blitz-robot"
|
||||
fi
|
||||
if [[ -z "${BLITZ_RUN_CONTEXT_FILE:-}" || "${BLITZ_RUN_CONTEXT_FILE}" == "${dev_runtime_dir}/run-context.env" ]]; then
|
||||
export BLITZ_RUN_CONTEXT_FILE="${BLITZ_RUNTIME_DIR}/run-context.env"
|
||||
fi
|
||||
if [[ -z "${BLITZ_RUN_ID_FILE:-}" || "${BLITZ_RUN_ID_FILE}" == "${dev_runtime_dir}/run-id" ]]; then
|
||||
export BLITZ_RUN_ID_FILE="${BLITZ_RUNTIME_DIR}/run-id"
|
||||
fi
|
||||
if [[ -z "${BLITZ_CURRENT_RUN_LINK:-}" || "${BLITZ_CURRENT_RUN_LINK}" == "${dev_run_root}/current" ]]; then
|
||||
export BLITZ_CURRENT_RUN_LINK="${BLITZ_RUN_ROOT}/current"
|
||||
fi
|
||||
|
||||
default_time_server="$(blitz_host_from_addr "${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}" || true)"
|
||||
|
||||
export BLITZ_BOOT_DELAY_SEC="${BLITZ_BOOT_DELAY_SEC:-30}"
|
||||
export BLITZ_RUN_ROOT="${BLITZ_RUN_ROOT:-/var/log/blitz-robot}"
|
||||
export BLITZ_LOG_FILE="${BLITZ_LOG_FILE:-/var/log/blitz-robot/startup.log}"
|
||||
export BLITZ_RUNTIME_DIR="${BLITZ_RUNTIME_DIR:-/run/blitz-robot}"
|
||||
export BLITZ_RUN_CONTEXT_FILE="${BLITZ_RUN_CONTEXT_FILE:-${BLITZ_RUNTIME_DIR}/run-context.env}"
|
||||
export BLITZ_RUN_ID_FILE="${BLITZ_RUN_ID_FILE:-${BLITZ_RUNTIME_DIR}/run-id}"
|
||||
export BLITZ_CURRENT_RUN_LINK="${BLITZ_CURRENT_RUN_LINK:-${BLITZ_RUN_ROOT}/current}"
|
||||
export BLITZ_5G_DIAL_DIR="${BLITZ_5G_DIAL_DIR:-${BOOT_SCRIPT_DIR}}"
|
||||
export BLITZ_5G_SERIAL_PORT="${BLITZ_5G_SERIAL_PORT:-/dev/ttyUSB7}"
|
||||
export BLITZ_5G_INTERFACE="${BLITZ_5G_INTERFACE:-}"
|
||||
export BLITZ_5G_MODEM_SUBNET="${BLITZ_5G_MODEM_SUBNET:-192.168.224.0/22}"
|
||||
export BLITZ_5G_GATEWAY="${BLITZ_5G_GATEWAY:-192.168.225.1}"
|
||||
export BLITZ_5G_SKIP_DHCP="${BLITZ_5G_SKIP_DHCP:-0}"
|
||||
export BLITZ_5G_REMOVE_DEFAULT_ROUTE="${BLITZ_5G_REMOVE_DEFAULT_ROUTE:-1}"
|
||||
export BLITZ_5G_ROUTE_TARGETS="${BLITZ_5G_ROUTE_TARGETS:-106.55.173.235}"
|
||||
export BLITZ_5G_INFO_JSON="${BLITZ_5G_INFO_JSON:-${BLITZ_5G_DIAL_DIR}/modem_network_info.json}"
|
||||
export BLITZ_5G_DISABLE_INTERFACES="${BLITZ_5G_DISABLE_INTERFACES:-}"
|
||||
export BLITZ_5G_SERIAL_WAIT_SEC="${BLITZ_5G_SERIAL_WAIT_SEC:-60}"
|
||||
export BLITZ_5G_ROUTE_WAIT_SEC="${BLITZ_5G_ROUTE_WAIT_SEC:-30}"
|
||||
export BLITZ_TIME_SERVER_IP="${BLITZ_TIME_SERVER_IP:-${default_time_server}}"
|
||||
export BLITZ_ROS_USER="${BLITZ_ROS_USER:-nvidia}"
|
||||
export BLITZ_ROS_SOCKET_WAIT_SEC="${BLITZ_ROS_SOCKET_WAIT_SEC:-20}"
|
||||
export BLITZ_WATCHDOG_INTERVAL_SEC="${BLITZ_WATCHDOG_INTERVAL_SEC:-5}"
|
||||
export BLITZ_HEALTH_STALE_SEC="${BLITZ_HEALTH_STALE_SEC:-15}"
|
||||
export BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC="${BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC:-15}"
|
||||
export BLITZ_KCP_STATS_INTERVAL_MS="${BLITZ_KCP_STATS_INTERVAL_MS:-1000}"
|
||||
export BLITZ_CONTROL_LATENCY_LOG_ENABLED="${BLITZ_CONTROL_LATENCY_LOG_ENABLED:-1}"
|
||||
export BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD="${BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD:-100}"
|
||||
export BLITZ_5G_LINK_LOG_INTERVAL_SEC="${BLITZ_5G_LINK_LOG_INTERVAL_SEC:-5}"
|
||||
export BLITZ_JSONL_FLUSH_INTERVAL_MS="${BLITZ_JSONL_FLUSH_INTERVAL_MS:-1000}"
|
||||
export BLITZ_JSONL_FLUSH_BYTES="${BLITZ_JSONL_FLUSH_BYTES:-262144}"
|
||||
export BLITZ_JSONL_ROTATE_BYTES="${BLITZ_JSONL_ROTATE_BYTES:-134217728}"
|
||||
export BLITZ_JSONL_ROTATE_FILES="${BLITZ_JSONL_ROTATE_FILES:-8}"
|
||||
export BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC="${BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC:-5}"
|
||||
export BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC="${BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC:-30}"
|
||||
export BLITZ_NETWORK_FAIL_THRESHOLD="${BLITZ_NETWORK_FAIL_THRESHOLD:-3}"
|
||||
export BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC="${BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC:-30}"
|
||||
export BLITZ_GPS_MONITOR_ENABLED="${BLITZ_GPS_MONITOR_ENABLED:-1}"
|
||||
export BLITZ_GPS_DEVICE_GLOB="${BLITZ_GPS_DEVICE_GLOB:-/dev/ttyCH341USB*}"
|
||||
export BLITZ_GPS_CHECK_INTERVAL_SEC="${BLITZ_GPS_CHECK_INTERVAL_SEC:-10}"
|
||||
export BLITZ_GPS_RESTART_UNITS="${BLITZ_GPS_RESTART_UNITS:-gpsd.socket gpsd.service}"
|
||||
export BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION="${BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION:-0}"
|
||||
export BLITZ_BOOT_ENV_LOADED="1"
|
||||
}
|
||||
|
||||
blitz_timestamp() {
|
||||
date '+%Y-%m-%d %H:%M:%S%z'
|
||||
}
|
||||
|
||||
blitz_sanitize_detail() {
|
||||
local detail="${1:-}"
|
||||
|
||||
detail="${detail//$'\n'/ ; }"
|
||||
detail="${detail//$'\r'/ }"
|
||||
printf '%s' "${detail}"
|
||||
}
|
||||
|
||||
blitz_log() {
|
||||
local step="${1:-unknown-step}"
|
||||
local action="${2:-unknown-action}"
|
||||
local result="${3:-info}"
|
||||
local details="${4:-}"
|
||||
local exit_code="${5:-0}"
|
||||
|
||||
printf '%s | %s | %s | %s | %s | %s\n' \
|
||||
"$(blitz_timestamp)" \
|
||||
"${step}" \
|
||||
"${action}" \
|
||||
"${result}" \
|
||||
"$(blitz_sanitize_detail "${details}")" \
|
||||
"${exit_code}"
|
||||
}
|
||||
|
||||
blitz_join_cmd() {
|
||||
local cmd=()
|
||||
local arg
|
||||
|
||||
for arg in "$@"; do
|
||||
cmd+=("$(printf '%q' "${arg}")")
|
||||
done
|
||||
printf '%s' "${cmd[*]}"
|
||||
}
|
||||
|
||||
blitz_require_command() {
|
||||
local command_name="$1"
|
||||
local step="${2:-precheck}"
|
||||
|
||||
if command -v "${command_name}" >/dev/null 2>&1; then
|
||||
blitz_log "${step}" "require-command" "success" "command=${command_name}" 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
blitz_log "${step}" "require-command" "failure" "missing command: ${command_name}" 127
|
||||
return 127
|
||||
}
|
||||
|
||||
blitz_require_file() {
|
||||
local path="$1"
|
||||
local step="${2:-precheck}"
|
||||
|
||||
if [[ -f "${path}" ]]; then
|
||||
blitz_log "${step}" "require-file" "success" "path=${path}" 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
blitz_log "${step}" "require-file" "failure" "missing file: ${path}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_require_executable() {
|
||||
local path="$1"
|
||||
local step="${2:-precheck}"
|
||||
|
||||
if [[ -x "${path}" ]]; then
|
||||
blitz_log "${step}" "require-executable" "success" "path=${path}" 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
blitz_log "${step}" "require-executable" "failure" "missing executable: ${path}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_require_root() {
|
||||
local step="${1:-precheck}"
|
||||
|
||||
if [[ "${EUID}" -eq 0 ]]; then
|
||||
blitz_log "${step}" "require-root" "success" "uid=${EUID}" 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
blitz_log "${step}" "require-root" "failure" "root privileges are required" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_run() {
|
||||
local step="$1"
|
||||
local action="$2"
|
||||
local rc
|
||||
shift 2
|
||||
|
||||
blitz_log "${step}" "${action}" "start" "$(blitz_join_cmd "$@")" 0
|
||||
if "$@"; then
|
||||
blitz_log "${step}" "${action}" "success" "$(blitz_join_cmd "$@")" 0
|
||||
return 0
|
||||
else
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
blitz_log "${step}" "${action}" "failure" "$(blitz_join_cmd "$@")" "${rc}"
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
blitz_route_ready() {
|
||||
local target_ip="$1"
|
||||
local expected_interface="${2:-}"
|
||||
local route_output
|
||||
|
||||
route_output="$(ip route get "${target_ip}" 2>&1 || true)"
|
||||
if [[ -z "${route_output}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${route_output}" == *"unreachable"* || "${route_output}" == *"prohibit"* ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "${expected_interface}" && "${route_output}" != *" dev ${expected_interface} "* && "${route_output}" != *" dev ${expected_interface}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "${route_output}"
|
||||
return 0
|
||||
}
|
||||
|
||||
blitz_interface_exists() {
|
||||
local interface_name="${1:-}"
|
||||
|
||||
if [[ -z "${interface_name}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
ip link show dev "${interface_name}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
blitz_read_5g_info_interface() {
|
||||
local info_json="$1"
|
||||
|
||||
if [[ -z "${info_json}" || ! -f "${info_json}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 - "${info_json}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except Exception:
|
||||
raise SystemExit(1)
|
||||
|
||||
interface = str(payload.get("interface") or "").strip()
|
||||
if not interface:
|
||||
raise SystemExit(1)
|
||||
|
||||
print(interface)
|
||||
PY
|
||||
}
|
||||
|
||||
blitz_detect_5g_interface_from_subnet() {
|
||||
local modem_subnet="${1:-${BLITZ_5G_MODEM_SUBNET:-}}"
|
||||
|
||||
if [[ -z "${modem_subnet}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 - "${modem_subnet}" <<'PY'
|
||||
import ipaddress
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
subnet = ipaddress.ip_network(sys.argv[1], strict=False)
|
||||
skip = {"lo", "docker0", "l4tbr0"}
|
||||
|
||||
def priority(name: str) -> tuple[int, str]:
|
||||
if name.startswith("enx"):
|
||||
return (0, name)
|
||||
if name.startswith("wwan"):
|
||||
return (1, name)
|
||||
if name.startswith("usb"):
|
||||
return (2, name)
|
||||
if name.startswith("eth"):
|
||||
return (3, name)
|
||||
return (9, name)
|
||||
|
||||
try:
|
||||
output = subprocess.check_output(["ip", "-j", "-4", "addr", "show"], text=True)
|
||||
payload = json.loads(output)
|
||||
except Exception:
|
||||
raise SystemExit(1)
|
||||
|
||||
candidates = []
|
||||
for item in payload:
|
||||
ifname = str(item.get("ifname") or "").strip()
|
||||
if not ifname or ifname in skip:
|
||||
continue
|
||||
for addr in item.get("addr_info") or []:
|
||||
if addr.get("family") != "inet":
|
||||
continue
|
||||
local = addr.get("local")
|
||||
prefixlen = addr.get("prefixlen")
|
||||
if not local or prefixlen is None:
|
||||
continue
|
||||
try:
|
||||
iface = ipaddress.ip_interface(f"{local}/{prefixlen}")
|
||||
except ValueError:
|
||||
continue
|
||||
if iface.ip in subnet:
|
||||
candidates.append((priority(ifname), ifname))
|
||||
break
|
||||
|
||||
if not candidates:
|
||||
raise SystemExit(1)
|
||||
|
||||
candidates.sort(key=lambda item: item[0])
|
||||
print(candidates[0][1])
|
||||
PY
|
||||
}
|
||||
|
||||
blitz_refresh_5g_info_json() {
|
||||
local interface_name="$1"
|
||||
local info_json="${2:-${BLITZ_5G_INFO_JSON:-}}"
|
||||
|
||||
if [[ -z "${interface_name}" || -z "${info_json}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 - "${interface_name}" "${info_json}" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
interface_name = sys.argv[1]
|
||||
path = sys.argv[2]
|
||||
|
||||
try:
|
||||
output = subprocess.check_output(["ip", "-j", "addr", "show", "dev", interface_name], text=True)
|
||||
payload = json.loads(output)
|
||||
except Exception:
|
||||
raise SystemExit(1)
|
||||
|
||||
if not payload:
|
||||
raise SystemExit(1)
|
||||
|
||||
item = payload[0]
|
||||
ipv4 = []
|
||||
ipv6 = []
|
||||
for addr in item.get("addr_info") or []:
|
||||
local = addr.get("local")
|
||||
prefixlen = addr.get("prefixlen")
|
||||
family = addr.get("family")
|
||||
if not local or prefixlen is None:
|
||||
continue
|
||||
entry = f"{local}/{prefixlen}"
|
||||
if family == "inet":
|
||||
ipv4.append(entry)
|
||||
elif family == "inet6":
|
||||
ipv6.append(entry)
|
||||
|
||||
data = {
|
||||
"interface": interface_name,
|
||||
"ipv4": ipv4,
|
||||
"ipv6": ipv6,
|
||||
}
|
||||
|
||||
parent = os.path.dirname(path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
temp_path = f"{path}.tmp.{os.getpid()}"
|
||||
with open(temp_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, ensure_ascii=False, indent=2)
|
||||
os.replace(temp_path, path)
|
||||
PY
|
||||
}
|
||||
|
||||
blitz_resolve_5g_interface() {
|
||||
local explicit_interface="${BLITZ_5G_INTERFACE:-}"
|
||||
local info_json="${BLITZ_5G_INFO_JSON:-}"
|
||||
local recorded_interface=""
|
||||
local detected_interface=""
|
||||
|
||||
if [[ -n "${explicit_interface}" ]]; then
|
||||
if blitz_interface_exists "${explicit_interface}"; then
|
||||
printf '%s\n' "${explicit_interface}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
recorded_interface="$(blitz_read_5g_info_interface "${info_json}" || true)"
|
||||
if [[ -n "${recorded_interface}" ]] && blitz_interface_exists "${recorded_interface}"; then
|
||||
printf '%s\n' "${recorded_interface}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
detected_interface="$(blitz_detect_5g_interface_from_subnet || true)"
|
||||
if [[ -n "${detected_interface}" ]]; then
|
||||
if [[ "${detected_interface}" != "${recorded_interface}" ]]; then
|
||||
blitz_refresh_5g_info_json "${detected_interface}" "${info_json}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
printf '%s\n' "${detected_interface}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_prepare_runtime_dir() {
|
||||
local runtime_dir
|
||||
|
||||
blitz_load_boot_env
|
||||
runtime_dir="${BLITZ_RUNTIME_DIR}"
|
||||
|
||||
mkdir -p "${runtime_dir}"
|
||||
if [[ "${EUID}" -eq 0 ]]; then
|
||||
chown "root:${BLITZ_ROS_USER}" "${runtime_dir}"
|
||||
chmod 0775 "${runtime_dir}"
|
||||
else
|
||||
chmod 0775 "${runtime_dir}" 2>/dev/null || true
|
||||
fi
|
||||
blitz_log "runtime-dir" "prepare" "success" "path=${runtime_dir}" 0
|
||||
}
|
||||
|
||||
blitz_prepare_run_root() {
|
||||
local run_root
|
||||
local run_dir
|
||||
local incidents_dir
|
||||
|
||||
blitz_load_boot_env
|
||||
run_root="${BLITZ_RUN_ROOT}"
|
||||
run_dir="${run_root}/runs"
|
||||
incidents_dir="${run_root}/incidents"
|
||||
|
||||
mkdir -p "${run_dir}" "${incidents_dir}"
|
||||
if [[ "${EUID}" -eq 0 ]]; then
|
||||
chown -R "root:${BLITZ_ROS_USER}" "${run_root}" 2>/dev/null || true
|
||||
chmod 0775 "${run_root}" "${run_dir}" "${incidents_dir}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
blitz_load_run_context_env() {
|
||||
local context_file="${1:-${BLITZ_RUN_CONTEXT_FILE:-}}"
|
||||
|
||||
if [[ -z "${context_file}" || ! -f "${context_file}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${context_file}"
|
||||
set +a
|
||||
return 0
|
||||
}
|
||||
|
||||
blitz_read_run_id() {
|
||||
local run_id_file="${BLITZ_RUN_ID_FILE:-}"
|
||||
|
||||
if [[ -z "${run_id_file}" || ! -f "${run_id_file}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
tr -d '\r\n' < "${run_id_file}"
|
||||
}
|
||||
|
||||
blitz_utc_compact_timestamp() {
|
||||
date -u '+%Y%m%dT%H%M%SZ'
|
||||
}
|
||||
|
||||
blitz_new_run_id() {
|
||||
printf '%s\n' "$(blitz_utc_compact_timestamp)"
|
||||
}
|
||||
|
||||
blitz_new_incident_id() {
|
||||
local prefix="${1:-incident}"
|
||||
printf '%s-%s-%d\n' "${prefix}" "$(blitz_utc_compact_timestamp)" "$$"
|
||||
}
|
||||
|
||||
blitz_new_instance_id() {
|
||||
printf '%s-%d\n' "$(blitz_utc_compact_timestamp)" "$$"
|
||||
}
|
||||
|
||||
blitz_git_commit() {
|
||||
git -C "${OMNISOCKETGO_ROOT}" rev-parse HEAD 2>/dev/null || true
|
||||
}
|
||||
|
||||
blitz_git_dirty_flag() {
|
||||
if git -C "${OMNISOCKETGO_ROOT}" diff --quiet --ignore-submodules=dirty >/dev/null 2>&1; then
|
||||
printf '0\n'
|
||||
return 0
|
||||
fi
|
||||
printf '1\n'
|
||||
}
|
||||
|
||||
blitz_write_run_context() {
|
||||
local run_id="$1"
|
||||
local run_dir="$2"
|
||||
local boot_id="$3"
|
||||
local context_file="${BLITZ_RUN_CONTEXT_FILE}"
|
||||
local id_file="${BLITZ_RUN_ID_FILE}"
|
||||
local temp_context
|
||||
local temp_info
|
||||
local commit_hash
|
||||
local dirty_flag
|
||||
local started_at
|
||||
|
||||
commit_hash="$(blitz_git_commit)"
|
||||
dirty_flag="$(blitz_git_dirty_flag)"
|
||||
started_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
temp_context="${context_file}.tmp.$$"
|
||||
temp_info="${run_dir}/run-info.json.tmp.$$"
|
||||
|
||||
mkdir -p "${run_dir}"
|
||||
printf '%s\n' "${run_id}" > "${id_file}"
|
||||
|
||||
cat > "${temp_context}" <<EOF
|
||||
BLITZ_RUN_ID=${run_id}
|
||||
BLITZ_RUN_DIR=${run_dir}
|
||||
BLITZ_BOOT_ID=${boot_id}
|
||||
BLITZ_RUN_ROOT=${BLITZ_RUN_ROOT}
|
||||
EOF
|
||||
mv -f "${temp_context}" "${context_file}"
|
||||
|
||||
python3 - "${temp_info}" "${run_id}" "${run_dir}" "${boot_id}" "${started_at}" "${commit_hash}" "${dirty_flag}" "${HOSTNAME:-$(hostname)}" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
path, run_id, run_dir, boot_id, started_at, commit_hash, dirty_flag, hostname = sys.argv[1:9]
|
||||
payload = {
|
||||
"run_id": run_id,
|
||||
"run_dir": run_dir,
|
||||
"boot_id": boot_id,
|
||||
"started_at": started_at,
|
||||
"hostname": hostname,
|
||||
"git_commit": commit_hash,
|
||||
"git_dirty": dirty_flag == "1",
|
||||
"env": {
|
||||
key: os.environ.get(key, "")
|
||||
for key in sorted(os.environ)
|
||||
if key.startswith(("BLITZ_", "OMNI_", "ROBOT_RECEIVER_"))
|
||||
},
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
PY
|
||||
mv -f "${temp_info}" "${run_dir}/run-info.json"
|
||||
ln -sfn "${run_dir}" "${BLITZ_CURRENT_RUN_LINK}"
|
||||
}
|
||||
|
||||
blitz_init_run_context() {
|
||||
local run_id
|
||||
local boot_id
|
||||
local run_dir
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_prepare_runtime_dir
|
||||
blitz_prepare_run_root
|
||||
|
||||
run_id="$(blitz_new_run_id)"
|
||||
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || blitz_new_run_id)"
|
||||
run_dir="${BLITZ_RUN_ROOT}/runs/${run_id}"
|
||||
|
||||
export BLITZ_RUN_ID="${run_id}"
|
||||
export BLITZ_RUN_DIR="${run_dir}"
|
||||
export BLITZ_BOOT_ID="${boot_id}"
|
||||
blitz_write_run_context "${run_id}" "${run_dir}" "${boot_id}"
|
||||
blitz_log "run-context" "init" "success" "run_id=${run_id} run_dir=${run_dir}" 0
|
||||
}
|
||||
|
||||
blitz_require_run_context() {
|
||||
blitz_load_boot_env
|
||||
if blitz_load_run_context_env; then
|
||||
return 0
|
||||
fi
|
||||
blitz_log "run-context" "load" "failure" "missing ${BLITZ_RUN_CONTEXT_FILE}" 1
|
||||
return 1
|
||||
}
|
||||
|
||||
blitz_ensure_instance_id() {
|
||||
if [[ -n "${BLITZ_INSTANCE_ID:-}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
export BLITZ_INSTANCE_ID="$(blitz_new_instance_id)"
|
||||
}
|
||||
|
||||
blitz_jsonl_rotate_if_needed() {
|
||||
local path="$1"
|
||||
local max_bytes="${2:-${BLITZ_JSONL_ROTATE_BYTES:-0}}"
|
||||
local max_files="${3:-${BLITZ_JSONL_ROTATE_FILES:-0}}"
|
||||
local size=0
|
||||
local index
|
||||
|
||||
if [[ -z "${path}" || ! -f "${path}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if (( max_bytes <= 0 || max_files <= 0 )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
size="$(stat -c %s "${path}" 2>/dev/null || echo 0)"
|
||||
if (( size < max_bytes )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for (( index=max_files; index>=1; index-- )); do
|
||||
if [[ "${index}" -eq "${max_files}" ]]; then
|
||||
rm -f "${path}.${index}"
|
||||
fi
|
||||
if [[ -f "${path}.${index}" ]]; then
|
||||
mv -f "${path}.${index}" "${path}.$(( index + 1 ))"
|
||||
fi
|
||||
done
|
||||
mv -f "${path}" "${path}.1"
|
||||
}
|
||||
|
||||
blitz_jsonl_append_line() {
|
||||
local path="$1"
|
||||
local line="$2"
|
||||
|
||||
mkdir -p "$(dirname "${path}")"
|
||||
blitz_jsonl_rotate_if_needed "${path}"
|
||||
printf '%s\n' "${line}" >> "${path}"
|
||||
}
|
||||
|
||||
blitz_launch_incident_capture() {
|
||||
local launch_script="${BOOT_SCRIPT_DIR}/blitz-incident-capture-launch.sh"
|
||||
|
||||
if [[ ! -f "${launch_script}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
/bin/bash "${launch_script}" "$@" >/dev/null 2>&1 || return 1
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="disable"
|
||||
SYSTEMD_DEST_DIR="/etc/systemd/system"
|
||||
UNITS=(
|
||||
"blitz-watchdog.service"
|
||||
"blitz-5g-link-logger.service"
|
||||
"blitz-b-side-omnid.service"
|
||||
"blitz-ros-receiver.service"
|
||||
"blitz-5g-dial.service"
|
||||
"blitz-run-context.service"
|
||||
"blitz-boot-gate.service"
|
||||
"blitz-robot.target"
|
||||
)
|
||||
|
||||
stop_unit_if_present() {
|
||||
local unit_name="$1"
|
||||
local unit_path="${SYSTEMD_DEST_DIR}/${unit_name}"
|
||||
|
||||
if [[ ! -f "${unit_path}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
blitz_run "${STEP}" "stop-unit" systemctl stop "${unit_name}" || true
|
||||
}
|
||||
|
||||
disable_unit_if_present() {
|
||||
local unit_name="$1"
|
||||
local unit_path="${SYSTEMD_DEST_DIR}/${unit_name}"
|
||||
|
||||
if [[ ! -f "${unit_path}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
blitz_run "${STEP}" "disable-unit" systemctl disable "${unit_name}" || true
|
||||
}
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "${STEP}"
|
||||
blitz_require_command systemctl "${STEP}"
|
||||
|
||||
for unit_name in "${UNITS[@]}"; do
|
||||
stop_unit_if_present "${unit_name}"
|
||||
done
|
||||
|
||||
for unit_name in "${UNITS[@]}"; do
|
||||
disable_unit_if_present "${unit_name}"
|
||||
done
|
||||
|
||||
blitz_log "${STEP}" "complete" "success" "boot chain stopped and disabled; next reboot will not auto-start blitz services" 0
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
SYSTEMD_TEMPLATE_DIR="${SCRIPT_DIR}/systemd"
|
||||
SYSTEMD_DEST_DIR="/etc/systemd/system"
|
||||
|
||||
render_template() {
|
||||
local template_path="$1"
|
||||
local output_path="$2"
|
||||
|
||||
sed \
|
||||
-e "s|@OMNISOCKETGO_ROOT@|${OMNISOCKETGO_ROOT}|g" \
|
||||
-e "s|@BLITZ_LOG_FILE@|${BLITZ_LOG_FILE}|g" \
|
||||
-e "s|@BLITZ_ROS_USER@|${BLITZ_ROS_USER}|g" \
|
||||
"${template_path}" > "${output_path}"
|
||||
}
|
||||
|
||||
install_unit() {
|
||||
local template_name="$1"
|
||||
local temp_output
|
||||
|
||||
temp_output="$(mktemp)"
|
||||
render_template "${SYSTEMD_TEMPLATE_DIR}/${template_name}" "${temp_output}"
|
||||
install -m 0644 "${temp_output}" "${SYSTEMD_DEST_DIR}/${template_name%.in}"
|
||||
rm -f "${temp_output}"
|
||||
blitz_log "install" "install-unit" "success" "unit=${SYSTEMD_DEST_DIR}/${template_name%.in}" 0
|
||||
}
|
||||
|
||||
remove_unit_if_present() {
|
||||
local unit_name="$1"
|
||||
local unit_path="${SYSTEMD_DEST_DIR}/${unit_name}"
|
||||
|
||||
if [[ ! -f "${unit_path}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
systemctl disable --now "${unit_name}" >/dev/null 2>&1 || true
|
||||
rm -f "${unit_path}"
|
||||
blitz_log "install" "remove-unit" "success" "unit=${unit_path}" 0
|
||||
}
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_root "install"
|
||||
blitz_require_command install "install"
|
||||
blitz_require_command systemctl "install"
|
||||
|
||||
mkdir -p "${SYSTEMD_DEST_DIR}"
|
||||
install -d -m 0755 "$(dirname "${BLITZ_LOG_FILE}")"
|
||||
touch "${BLITZ_LOG_FILE}"
|
||||
chmod 0644 "${BLITZ_LOG_FILE}"
|
||||
blitz_log "install" "prepare-log-file" "success" "log_file=${BLITZ_LOG_FILE}" 0
|
||||
blitz_prepare_runtime_dir
|
||||
blitz_prepare_run_root
|
||||
|
||||
install_unit "blitz-boot-gate.service.in"
|
||||
install_unit "blitz-run-context.service.in"
|
||||
install_unit "blitz-5g-dial.service.in"
|
||||
install_unit "blitz-5g-link-logger.service.in"
|
||||
install_unit "blitz-ros-receiver.service.in"
|
||||
install_unit "blitz-b-side-omnid.service.in"
|
||||
install_unit "blitz-watchdog.service.in"
|
||||
install_unit "blitz-robot.target.in"
|
||||
remove_unit_if_present "blitz-time-sync.service"
|
||||
|
||||
blitz_run "install" "daemon-reload" systemctl daemon-reload
|
||||
blitz_run "install" "enable-target" systemctl enable blitz-robot.target
|
||||
blitz_log "install" "complete" "success" "run systemctl start blitz-robot.target to launch immediately" 0
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="runtime-dir"
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_prepare_runtime_dir
|
||||
blitz_log "${STEP}" "complete" "success" "runtime_dir=${BLITZ_RUNTIME_DIR}" 0
|
||||
854
robot/v4l2/OmniSocketGo_robot/scripts/boot/rndis_dial.py
Normal file
854
robot/v4l2/OmniSocketGo_robot/scripts/boot/rndis_dial.py
Normal file
@@ -0,0 +1,854 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RM520N-GL RNDIS 自动拨号脚本。
|
||||
|
||||
流程:
|
||||
1. 检测 USB 设备是否存在
|
||||
2. 打开 AT 口并检查 SIM 状态
|
||||
3. 配置 RNDIS 模式: AT+QCFG="usbnet",3
|
||||
4. 重启模块: AT+CFUN=1,1
|
||||
5. 等待模块重新枚举并识别 5G 网卡
|
||||
6. 如果网卡还没有 IPv4, 自动尝试 DHCP
|
||||
|
||||
用法:
|
||||
sudo python3 rndis_dial.py
|
||||
sudo python3 rndis_dial.py --serial-port /dev/ttyUSB7
|
||||
sudo python3 rndis_dial.py --interface eth0 #指定网口
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
import tty
|
||||
|
||||
USB_ID = "2c7c:0801"
|
||||
DEFAULT_SERIAL_PORT = "/dev/ttyUSB7" #串口设备节点
|
||||
DEFAULT_BAUD_RATE = 115200
|
||||
CHECK_INTERVAL = 2
|
||||
SERIAL_READ_TIMEOUT = 0.2
|
||||
SERIAL_POLL_INTERVAL = 0.1
|
||||
SERIAL_SETTLE_DELAY = 0.3
|
||||
AT_SYNC_RETRIES = 3
|
||||
AT_SYNC_TIMEOUT = 2.5
|
||||
# 示例地址 192.168.225.38/22 所在网段。
|
||||
# 拨号成功后会用这个网段来最终确认哪个接口是 5G 模组。
|
||||
DEFAULT_MODEM_SUBNET = "192.168.224.0/22"
|
||||
DEFAULT_MODEM_GATEWAY = "192.168.225.1"
|
||||
DEFAULT_PUBLIC_TARGETS = ("81.70.156.140", "106.55.173.235")
|
||||
DEFAULT_INFO_JSON = "modem_network_info.json"
|
||||
SKIP_INTERFACES = {"lo", "docker0", "l4tbr0"}
|
||||
BAUD_RATE_MAP = {
|
||||
9600: termios.B9600,
|
||||
19200: termios.B19200,
|
||||
38400: termios.B38400,
|
||||
57600: termios.B57600,
|
||||
115200: termios.B115200,
|
||||
}
|
||||
|
||||
|
||||
def run_cmd(cmd, timeout=30, check=False):
|
||||
print(f"[CMD] {format_shell_cmd(cmd)}")
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
output = (result.stdout or "") + (result.stderr or "")
|
||||
if check and result.returncode != 0:
|
||||
raise RuntimeError(f"命令执行失败: {' '.join(cmd)}\n{output.strip()}")
|
||||
return result.returncode, output.strip()
|
||||
|
||||
|
||||
def format_shell_cmd(cmd):
|
||||
"""把命令参数格式化成可直接阅读的 shell 形式。"""
|
||||
return " ".join(shlex.quote(part) for part in cmd)
|
||||
|
||||
|
||||
def parse_ipv4_address(value):
|
||||
try:
|
||||
return str(ipaddress.IPv4Address(value))
|
||||
except ipaddress.AddressValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"无效的 IPv4 地址: {value}") from exc
|
||||
|
||||
|
||||
def dedupe_keep_order(values):
|
||||
seen = set()
|
||||
result = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
result.append(value)
|
||||
return result
|
||||
|
||||
|
||||
def require_root():
|
||||
if os.geteuid() != 0:
|
||||
print("[FAIL] 请使用 sudo 运行此脚本")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def require_commands():
|
||||
missing = [cmd for cmd in ("lsusb", "ip") if shutil.which(cmd) is None]
|
||||
if missing:
|
||||
print(f"[FAIL] 缺少系统命令: {', '.join(missing)}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def usb_device_present():
|
||||
# 1. 第一次检测 lsusb，确认模块已经被系统识别。
|
||||
"""通过 lsusb 检查模块是否已经被系统识别。"""
|
||||
code, output = run_cmd(["lsusb"], timeout=10)
|
||||
if code != 0:
|
||||
return False, output
|
||||
|
||||
for line in output.splitlines():
|
||||
if USB_ID in line:
|
||||
return True, line.strip()
|
||||
return False, output
|
||||
|
||||
|
||||
def wait_for_usb_device(expected_present, timeout):
|
||||
"""等待模块 USB 设备下线或重新上线。"""
|
||||
deadline = time.time() + timeout
|
||||
last_seen = ""
|
||||
while time.time() < deadline:
|
||||
present, detail = usb_device_present()
|
||||
last_seen = detail
|
||||
if present == expected_present:
|
||||
return True, detail
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
return False, last_seen
|
||||
|
||||
|
||||
def wait_for_path(path, timeout):
|
||||
"""等待串口节点或其他路径重新出现。"""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if os.path.exists(path):
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def normalize_serial_output(text):
|
||||
"""整理串口原始输出，便于后续匹配关键字。"""
|
||||
cleaned = text.replace("\r", "\n")
|
||||
return "\n".join(line for line in cleaned.splitlines() if line.strip()).strip()
|
||||
|
||||
|
||||
def serial_response_complete(text):
|
||||
if not text:
|
||||
return False
|
||||
|
||||
for line in reversed(text.splitlines()):
|
||||
stripped = line.strip()
|
||||
if stripped == "OK":
|
||||
return True
|
||||
if "ERROR" in stripped:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class RawSerialSession:
|
||||
"""使用 Python 标准库直接控制 Linux 串口，尽量贴近 stty/raw 行为。"""
|
||||
|
||||
def __init__(self, port, baudrate):
|
||||
if baudrate not in BAUD_RATE_MAP:
|
||||
raise RuntimeError(f"不支持的波特率: {baudrate}")
|
||||
|
||||
self.port = port
|
||||
self.fd = None
|
||||
self._original_attrs = None
|
||||
|
||||
try:
|
||||
self.fd = os.open(port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
|
||||
self._original_attrs = termios.tcgetattr(self.fd)
|
||||
tty.setraw(self.fd, when=termios.TCSANOW)
|
||||
|
||||
attrs = termios.tcgetattr(self.fd)
|
||||
attrs[0] = 0
|
||||
attrs[1] = 0
|
||||
attrs[2] &= ~(termios.PARENB | termios.CSTOPB | termios.CSIZE)
|
||||
attrs[2] |= termios.CS8 | termios.CLOCAL | termios.CREAD
|
||||
attrs[3] = 0
|
||||
attrs[4] = BAUD_RATE_MAP[baudrate]
|
||||
attrs[5] = BAUD_RATE_MAP[baudrate]
|
||||
attrs[6][termios.VMIN] = 0
|
||||
attrs[6][termios.VTIME] = 0
|
||||
termios.tcsetattr(self.fd, termios.TCSANOW, attrs)
|
||||
termios.tcflush(self.fd, termios.TCIOFLUSH)
|
||||
except OSError as exc:
|
||||
self.close()
|
||||
raise RuntimeError(f"无法打开串口 {port}: {exc}") from exc
|
||||
|
||||
@property
|
||||
def is_open(self):
|
||||
return self.fd is not None
|
||||
|
||||
def reset_input_buffer(self):
|
||||
if self.fd is not None:
|
||||
termios.tcflush(self.fd, termios.TCIFLUSH)
|
||||
|
||||
def reset_output_buffer(self):
|
||||
if self.fd is not None:
|
||||
termios.tcflush(self.fd, termios.TCOFLUSH)
|
||||
|
||||
def write(self, data):
|
||||
if self.fd is None:
|
||||
raise OSError("串口未打开")
|
||||
|
||||
sent = 0
|
||||
while sent < len(data):
|
||||
try:
|
||||
written = os.write(self.fd, data[sent:])
|
||||
except BlockingIOError:
|
||||
time.sleep(SERIAL_POLL_INTERVAL)
|
||||
continue
|
||||
if written <= 0:
|
||||
raise OSError("串口写入返回 0 字节")
|
||||
sent += written
|
||||
|
||||
def flush(self):
|
||||
if self.fd is not None:
|
||||
termios.tcdrain(self.fd)
|
||||
|
||||
def read_chunk(self, timeout, size=4096):
|
||||
if self.fd is None:
|
||||
return b""
|
||||
|
||||
ready, _, _ = select.select([self.fd], [], [], timeout)
|
||||
if not ready:
|
||||
return b""
|
||||
|
||||
try:
|
||||
return os.read(self.fd, size)
|
||||
except BlockingIOError:
|
||||
return b""
|
||||
|
||||
def close(self):
|
||||
if self.fd is None:
|
||||
return
|
||||
|
||||
fd = self.fd
|
||||
self.fd = None
|
||||
|
||||
if self._original_attrs is not None:
|
||||
try:
|
||||
termios.tcsetattr(fd, termios.TCSANOW, self._original_attrs)
|
||||
except termios.error:
|
||||
pass
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def read_serial_output(session, timeout, allow_disconnect=False):
|
||||
"""在给定时间窗口内读取 AT 响应，直到出现结束标记或超时。"""
|
||||
deadline = time.time() + timeout
|
||||
chunks = []
|
||||
saw_terminal_line = False
|
||||
last_data_time = None
|
||||
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
chunk = session.read_chunk(timeout=min(SERIAL_READ_TIMEOUT, max(deadline - time.time(), 0)))
|
||||
except OSError as exc:
|
||||
if allow_disconnect and exc.errno in (errno.EIO, errno.ENODEV, errno.EBADF):
|
||||
break
|
||||
raise RuntimeError(f"读取串口响应失败: {exc}") from exc
|
||||
|
||||
if chunk:
|
||||
chunks.append(chunk.decode(errors="ignore"))
|
||||
last_data_time = time.time()
|
||||
current_text = normalize_serial_output("".join(chunks))
|
||||
if serial_response_complete(current_text):
|
||||
saw_terminal_line = True
|
||||
continue
|
||||
|
||||
if saw_terminal_line and last_data_time is not None and time.time() - last_data_time >= SERIAL_SETTLE_DELAY:
|
||||
break
|
||||
|
||||
time.sleep(SERIAL_POLL_INTERVAL)
|
||||
|
||||
return normalize_serial_output("".join(chunks))
|
||||
|
||||
|
||||
def open_serial_session(port):
|
||||
"""打开 AT 串口会话，后续在同一连接里顺序发送多条命令。"""
|
||||
ser = RawSerialSession(port=port, baudrate=DEFAULT_BAUD_RATE)
|
||||
time.sleep(0.2)
|
||||
ser.reset_input_buffer()
|
||||
ser.reset_output_buffer()
|
||||
return ser
|
||||
|
||||
|
||||
def execute_serial_step(ser, command, expect=None, timeout=3, allow_disconnect=False):
|
||||
"""在当前串口会话里发送一条 AT 命令并校验响应。"""
|
||||
print(f"[AT] {command}")
|
||||
try:
|
||||
ser.reset_input_buffer()
|
||||
ser.write((command + "\r").encode())
|
||||
ser.flush()
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f"AT 命令 `{command}` 发送失败: {exc}") from exc
|
||||
|
||||
response = read_serial_output(ser, timeout=timeout, allow_disconnect=allow_disconnect)
|
||||
|
||||
if response:
|
||||
print(response)
|
||||
else:
|
||||
print("(无响应)")
|
||||
|
||||
if "ERROR" in response:
|
||||
raise RuntimeError(f"AT 命令 `{command}` 执行失败: {response}")
|
||||
if expect and expect not in response and not allow_disconnect:
|
||||
raise RuntimeError(f"AT 命令 `{command}` 响应异常: {response or '空响应'}")
|
||||
return response
|
||||
|
||||
|
||||
def synchronize_at_channel(ser):
|
||||
"""某些模组 AT 口在刚打开时需要先用 AT 做一次预热。"""
|
||||
last_error = None
|
||||
|
||||
for attempt in range(1, AT_SYNC_RETRIES + 1):
|
||||
try:
|
||||
print(f"[INFO] 预热 AT 通道，第 {attempt} 次")
|
||||
response = execute_serial_step(ser, "AT", expect="OK", timeout=AT_SYNC_TIMEOUT)
|
||||
if "OK" in response:
|
||||
return
|
||||
except RuntimeError as exc:
|
||||
last_error = exc
|
||||
time.sleep(0.5)
|
||||
|
||||
if last_error is not None:
|
||||
raise RuntimeError(
|
||||
"AT 通道预热失败，请确认串口是否是 AT 命令口，例如 /dev/ttyUSB2"
|
||||
) from last_error
|
||||
raise RuntimeError("AT 通道预热失败")
|
||||
|
||||
|
||||
def run_serial_steps(port, steps):
|
||||
"""在同一个串口会话里顺序执行多条 AT 命令。"""
|
||||
ser = None
|
||||
|
||||
try:
|
||||
ser = open_serial_session(port)
|
||||
synchronize_at_channel(ser)
|
||||
for step in steps:
|
||||
execute_serial_step(
|
||||
ser,
|
||||
step["command"],
|
||||
expect=step.get("expect"),
|
||||
timeout=step.get("timeout", 3),
|
||||
allow_disconnect=step.get("allow_disconnect", False),
|
||||
)
|
||||
finally:
|
||||
if ser is not None and ser.is_open:
|
||||
ser.close()
|
||||
|
||||
def configure_rndis(port):
|
||||
# 2. 用 Python 串口库在同一会话里顺序执行拨号相关 AT 命令。
|
||||
"""切换到 RNDIS 模式并触发模块重启。"""
|
||||
if not wait_for_path(port, timeout=30):
|
||||
raise RuntimeError(f"串口不存在: {port}")
|
||||
|
||||
print(f"[OK] 串口已打开: {port}")
|
||||
run_serial_steps(
|
||||
port,
|
||||
[
|
||||
{"command": "AT+CPIN?", "expect": "READY", "timeout": 4},
|
||||
{"command": 'AT+QCFG="usbnet",3', "expect": "OK", "timeout": 5},
|
||||
{"command": "AT+CFUN=1,1", "timeout": 4, "allow_disconnect": True},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def get_interfaces():
|
||||
"""列出当前系统中的接口，过滤明显无关的本地接口。"""
|
||||
interfaces = []
|
||||
try:
|
||||
for name in os.listdir("/sys/class/net"):
|
||||
if name in SKIP_INTERFACES or is_usb_gadget(name):
|
||||
continue
|
||||
interfaces.append(name)
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
return sorted(interfaces)
|
||||
|
||||
|
||||
def is_usb_gadget(iface):
|
||||
"""过滤 Jetson 自己暴露出去的 gadget 网卡。"""
|
||||
sysfs_path = f"/sys/class/net/{iface}"
|
||||
if not os.path.exists(sysfs_path):
|
||||
return False
|
||||
return "/gadget/" in os.path.realpath(sysfs_path)
|
||||
|
||||
|
||||
def is_usb_network_interface(iface):
|
||||
"""判断接口是否来自 USB 设备。"""
|
||||
device_path = f"/sys/class/net/{iface}/device"
|
||||
if not os.path.exists(device_path):
|
||||
return False
|
||||
real_path = os.path.realpath(device_path)
|
||||
return "/usb" in real_path
|
||||
|
||||
|
||||
def get_ipv4_addrs():
|
||||
"""返回所有接口的 IPv4/CIDR 信息。"""
|
||||
code, output = run_cmd(["ip", "-o", "-4", "addr", "show"], timeout=10)
|
||||
if code != 0:
|
||||
return {}
|
||||
|
||||
ipv4_addrs = {}
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 4:
|
||||
iface = parts[1]
|
||||
ipv4_addrs.setdefault(iface, []).append(parts[3])
|
||||
return ipv4_addrs
|
||||
|
||||
|
||||
def get_ipv6_addrs():
|
||||
"""返回所有接口的 IPv6/CIDR 信息。"""
|
||||
code, output = run_cmd(["ip", "-o", "-6", "addr", "show"], timeout=10)
|
||||
if code != 0:
|
||||
return {}
|
||||
|
||||
ipv6_addrs = {}
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 4:
|
||||
iface = parts[1]
|
||||
ipv6_addrs.setdefault(iface, []).append(parts[3])
|
||||
return ipv6_addrs
|
||||
|
||||
|
||||
def interface_priority(iface):
|
||||
if iface.startswith("wwan"):
|
||||
return 0
|
||||
if iface.startswith("enx"):
|
||||
return 1
|
||||
if iface.startswith("usb"):
|
||||
return 2
|
||||
return 10
|
||||
|
||||
|
||||
def list_usb_network_candidates(explicit_iface=None):
|
||||
"""列出拨号前可尝试的 USB 网卡候选项。
|
||||
|
||||
这里不靠固定网口名确认 5G 模组，只是在还没有 IP 的时候先缩小范围。
|
||||
真正确认模组接口，会在 DHCP 之后根据 IP 网段判断。
|
||||
"""
|
||||
candidates = []
|
||||
|
||||
for iface in get_interfaces():
|
||||
if explicit_iface and iface != explicit_iface:
|
||||
continue
|
||||
if not is_usb_network_interface(iface):
|
||||
continue
|
||||
candidates.append((interface_priority(iface), iface))
|
||||
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
candidates.sort()
|
||||
return [iface for _, iface in candidates]
|
||||
|
||||
|
||||
def ip_in_subnet(ip_cidr, subnet):
|
||||
"""判断接口地址是否落在指定网段内。"""
|
||||
try:
|
||||
return ipaddress.ip_interface(ip_cidr).ip in ipaddress.ip_network(subnet, strict=False)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def find_interface_by_subnet(modem_subnet, explicit_iface=None):
|
||||
"""拨号成功后，通过 IP 网段确认 5G 模组网卡。"""
|
||||
candidates = []
|
||||
for iface, addrs in get_ipv4_addrs().items():
|
||||
if iface in SKIP_INTERFACES or is_usb_gadget(iface):
|
||||
continue
|
||||
if not is_usb_network_interface(iface):
|
||||
continue
|
||||
if explicit_iface and iface != explicit_iface:
|
||||
continue
|
||||
|
||||
matched_addrs = [addr for addr in addrs if ip_in_subnet(addr, modem_subnet)]
|
||||
if matched_addrs:
|
||||
candidates.append((interface_priority(iface), iface, matched_addrs))
|
||||
|
||||
if not candidates:
|
||||
return None, []
|
||||
|
||||
candidates.sort()
|
||||
_, iface, matched_addrs = candidates[0]
|
||||
return iface, matched_addrs
|
||||
|
||||
|
||||
def wait_for_usb_candidates(explicit_iface=None, timeout=90):
|
||||
"""等待模块枚举出 USB 网卡候选项。"""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
candidates = list_usb_network_candidates(explicit_iface=explicit_iface)
|
||||
if candidates:
|
||||
return candidates
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
return []
|
||||
|
||||
|
||||
def bring_interface_up(iface):
|
||||
code, output = run_cmd(["ip", "link", "set", "dev", iface, "up"], timeout=10)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"拉起网卡失败: {iface}\n{output}")
|
||||
|
||||
|
||||
def renew_dhcp(iface):
|
||||
dhclient = shutil.which("dhclient")
|
||||
udhcpc = shutil.which("udhcpc")
|
||||
|
||||
if dhclient:
|
||||
print(f"[INFO] 使用 dhclient 为 {iface} 获取 IP")
|
||||
code, output = run_cmd(["dhclient", "-1", "-v", iface], timeout=45)
|
||||
return code == 0, output
|
||||
|
||||
if udhcpc:
|
||||
print(f"[INFO] 使用 udhcpc 为 {iface} 获取 IP")
|
||||
code, output = run_cmd(["udhcpc", "-n", "-q", "-i", iface], timeout=45)
|
||||
return code == 0, output
|
||||
|
||||
return False, "系统中未找到 dhclient 或 udhcpc"
|
||||
|
||||
|
||||
def get_default_routes(iface):
|
||||
code, output = run_cmd(["ip", "-o", "route", "show", "default", "dev", iface], timeout=10)
|
||||
if code != 0:
|
||||
return []
|
||||
return [line.strip() for line in output.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def resolve_gateway(iface, fallback_gateway):
|
||||
for route in get_default_routes(iface):
|
||||
tokens = route.split()
|
||||
for index, token in enumerate(tokens[:-1]):
|
||||
if token == "via":
|
||||
gateway = tokens[index + 1]
|
||||
print(f"[INFO] 从默认路由检测到 {iface} 网关: {gateway}")
|
||||
return gateway
|
||||
|
||||
print(f"[INFO] 未从默认路由检测到 {iface} 网关，回退到 {fallback_gateway}")
|
||||
return fallback_gateway
|
||||
|
||||
|
||||
def delete_default_routes(iface):
|
||||
removed = 0
|
||||
|
||||
while True:
|
||||
routes = get_default_routes(iface)
|
||||
if not routes:
|
||||
return removed
|
||||
|
||||
deleted_this_round = False
|
||||
for route in routes:
|
||||
cmd = ["ip", "route", "del", *route.split()]
|
||||
code, output = run_cmd(cmd, timeout=10)
|
||||
if code != 0:
|
||||
code, output = run_cmd(["ip", "route", "del", "default", "dev", iface], timeout=10)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"删除默认路由失败: {iface}\n{output}")
|
||||
removed += 1
|
||||
deleted_this_round = True
|
||||
|
||||
if not deleted_this_round:
|
||||
raise RuntimeError(f"未能删除 {iface} 的默认路由")
|
||||
|
||||
|
||||
def install_host_routes(iface, gateway, targets):
|
||||
for target in dedupe_keep_order(targets):
|
||||
cmd = ["ip", "route", "replace", f"{target}/32", "via", gateway, "dev", iface]
|
||||
code, output = run_cmd(cmd, timeout=10)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"添加主机路由失败: {target} via {gateway} dev {iface}\n{output}")
|
||||
|
||||
print(f"[OK] 已添加主机路由: {target}/32 via {gateway} dev {iface}")
|
||||
|
||||
|
||||
def enforce_route_policy(iface, fallback_gateway, route_targets):
|
||||
gateway = resolve_gateway(iface, fallback_gateway)
|
||||
removed = delete_default_routes(iface)
|
||||
print(f"[OK] 已删除 {iface} 上的 {removed} 条默认路由")
|
||||
|
||||
if route_targets:
|
||||
install_host_routes(iface, gateway, route_targets)
|
||||
else:
|
||||
print(f"[WARN] {iface} 未配置任何主机路由目标，5G 将不再承载公网流量")
|
||||
|
||||
|
||||
def ensure_ipv4(iface):
|
||||
"""为指定接口申请 IPv4 地址。"""
|
||||
ipv4_addrs = get_ipv4_addrs().get(iface, [])
|
||||
if ipv4_addrs:
|
||||
return ipv4_addrs
|
||||
|
||||
bring_interface_up(iface)
|
||||
ok, output = renew_dhcp(iface)
|
||||
if output:
|
||||
print(output)
|
||||
if not ok:
|
||||
return []
|
||||
|
||||
return get_ipv4_addrs().get(iface, [])
|
||||
|
||||
|
||||
def acquire_modem_interface(modem_subnet, explicit_iface=None):
|
||||
"""通过 DHCP + IP 网段识别真正的模组接口。"""
|
||||
iface, matched_addrs = find_interface_by_subnet(
|
||||
modem_subnet,
|
||||
explicit_iface=explicit_iface,
|
||||
)
|
||||
if iface:
|
||||
return iface, matched_addrs
|
||||
|
||||
candidates = list_usb_network_candidates(explicit_iface=explicit_iface)
|
||||
if not candidates:
|
||||
raise RuntimeError("未找到可尝试 DHCP 的 USB 网卡候选项")
|
||||
|
||||
print(f"[INFO] 当前 USB 网卡候选项: {', '.join(candidates)}")
|
||||
|
||||
for iface in candidates:
|
||||
print(f"[INFO] 尝试为 {iface} 获取 IPv4")
|
||||
ensure_ipv4(iface)
|
||||
|
||||
matched_iface, matched_addrs = find_interface_by_subnet(
|
||||
modem_subnet,
|
||||
explicit_iface=explicit_iface,
|
||||
)
|
||||
if matched_iface:
|
||||
return matched_iface, matched_addrs
|
||||
|
||||
return None, []
|
||||
|
||||
|
||||
def print_interface_status(iface):
|
||||
# 3. 拨号成功后，打印 ip/ifconfig，确认模组网口和地址。
|
||||
print(f"[OK] 检测到 5G 网卡: {iface}")
|
||||
|
||||
code, output = run_cmd(["ip", "-4", "addr", "show", "dev", iface], timeout=10)
|
||||
if code == 0 and output:
|
||||
print(output)
|
||||
|
||||
if shutil.which("ifconfig"):
|
||||
code, ifconfig_output = run_cmd(["ifconfig", iface], timeout=10)
|
||||
if code == 0 and ifconfig_output:
|
||||
print("\n===== ifconfig =====")
|
||||
print(ifconfig_output)
|
||||
|
||||
|
||||
def save_interface_info(iface, output_file=DEFAULT_INFO_JSON):
|
||||
"""把网口名称、IPv4、IPv6 保存到 JSON 文件。"""
|
||||
data = {
|
||||
"interface": iface,
|
||||
"ipv4": get_ipv4_addrs().get(iface, []),
|
||||
"ipv6": get_ipv6_addrs().get(iface, []),
|
||||
}
|
||||
|
||||
with open(output_file, "w", encoding="utf-8") as json_file:
|
||||
json.dump(data, json_file, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"[OK] 网口信息已保存到 {output_file}")
|
||||
|
||||
|
||||
def ping_target(iface, target, count=3, timeout=15):
|
||||
"""通过指定网口 ping 一个目标。"""
|
||||
code, output = run_cmd(
|
||||
["ping", "-I", iface, "-c", str(count), "-W", "3", target],
|
||||
timeout=timeout,
|
||||
)
|
||||
return code == 0, output
|
||||
|
||||
|
||||
def print_ping_summary(output):
|
||||
"""只打印 ping 的关键结果。"""
|
||||
for line in output.splitlines():
|
||||
if "packets transmitted" in line or "rtt " in line or "Destination " in line:
|
||||
print(line)
|
||||
|
||||
|
||||
def verify_connectivity(iface, gateway=DEFAULT_MODEM_GATEWAY, targets=DEFAULT_PUBLIC_TARGETS, retry_interval=3, max_wait=45):
|
||||
# 4. 最后先 ping 模组网关，再重试公网连通性。
|
||||
"""先测模组网关，再轮询公网目标地址。"""
|
||||
ok, output = ping_target(iface, gateway, count=3, timeout=15)
|
||||
if ok:
|
||||
print(f"[OK] {iface} 可到达模组网关 {gateway}")
|
||||
print_ping_summary(output)
|
||||
else:
|
||||
print(f"[WARN] {iface} 无法到达模组网关 {gateway}")
|
||||
if output:
|
||||
print(output)
|
||||
return False
|
||||
|
||||
deadline = time.time() + max_wait
|
||||
attempt = 1
|
||||
while True:
|
||||
for target in targets:
|
||||
ok, output = ping_target(iface, target, count=3, timeout=15)
|
||||
if ok:
|
||||
print(f"[OK] {iface} 可通过 {target}")
|
||||
print_ping_summary(output)
|
||||
return True
|
||||
|
||||
print(f"[WARN] 第 {attempt} 次 Ping {target} 失败")
|
||||
if output:
|
||||
print_ping_summary(output)
|
||||
|
||||
if time.time() >= deadline:
|
||||
print(f"[WARN] {iface} 在 {max_wait} 秒内仍无法连通 {', '.join(targets)}")
|
||||
return False
|
||||
|
||||
attempt += 1
|
||||
time.sleep(retry_interval)
|
||||
|
||||
|
||||
def ping_via_interface(iface, targets=DEFAULT_PUBLIC_TARGETS):
|
||||
"""保留原调用点，内部走完整连通性检查。"""
|
||||
return verify_connectivity(iface, targets=targets)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="RM520N-GL RNDIS 自动拨号脚本")
|
||||
parser.add_argument(
|
||||
"--serial-port",
|
||||
default=DEFAULT_SERIAL_PORT,
|
||||
help=f"AT 串口路径，默认 {DEFAULT_SERIAL_PORT}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interface",
|
||||
help="指定期望的 5G 网卡名，例如 eth0",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modem-subnet",
|
||||
default=DEFAULT_MODEM_SUBNET,
|
||||
help=f"拨号成功后用于识别模组接口的 IPv4 网段，默认 {DEFAULT_MODEM_SUBNET}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gateway",
|
||||
type=parse_ipv4_address,
|
||||
default=DEFAULT_MODEM_GATEWAY,
|
||||
help=f"5G 模组网关地址，默认 {DEFAULT_MODEM_GATEWAY}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-dhcp",
|
||||
action="store_true",
|
||||
help="只等待 USB 网卡出现，不主动申请 IPv4",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--remove-default-route",
|
||||
action="store_true",
|
||||
help="拨号成功后删除 5G 接口上的默认路由，只保留显式主机路由",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--route-target",
|
||||
action="append",
|
||||
default=[],
|
||||
type=parse_ipv4_address,
|
||||
help="拨号完成后通过 5G 接口保留的 IPv4 主机路由目标，可重复传入",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
require_root()
|
||||
require_commands()
|
||||
|
||||
print("===== RM520N-GL RNDIS 自动拨号 =====")
|
||||
print(f"[INFO] 目标模组网段: {args.modem_subnet}")
|
||||
|
||||
#1.检测 lsusb，确认是否识别到模块
|
||||
present, detail = usb_device_present()
|
||||
if not present:
|
||||
print(f"[FAIL] 未检测到模块 USB 设备 {USB_ID}")
|
||||
if detail:
|
||||
print(detail)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[OK] 检测到 USB 设备: {detail}")
|
||||
print(f"[INFO] 使用 AT 口: {args.serial_port}")
|
||||
|
||||
#2.进行 Python 串口拨号
|
||||
try:
|
||||
configure_rndis(args.serial_port)
|
||||
|
||||
print("[INFO] 已发送 AT+CFUN=1,1，等待模块重启")
|
||||
disappeared, _ = wait_for_usb_device(expected_present=False, timeout=25)
|
||||
if disappeared:
|
||||
print("[OK] 模块已下线，继续等待重新枚举")
|
||||
else:
|
||||
print("[WARN] 未观察到模块下线，继续等待重新枚举")
|
||||
|
||||
reappeared, detail = wait_for_usb_device(expected_present=True, timeout=90)
|
||||
if not reappeared:
|
||||
print(f"[FAIL] 模块重启后未重新枚举: {USB_ID}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[OK] 模块已重新枚举: {detail}")
|
||||
|
||||
candidates = wait_for_usb_candidates(explicit_iface=args.interface, timeout=90)
|
||||
if not candidates:
|
||||
print("[FAIL] 未检测到 5G 模组枚举出的 USB 网卡")
|
||||
sys.exit(1)
|
||||
|
||||
if args.skip_dhcp:
|
||||
print(f"[INFO] 当前 USB 网卡候选项: {', '.join(candidates)}")
|
||||
iface, ipv4_addrs = find_interface_by_subnet(
|
||||
args.modem_subnet,
|
||||
explicit_iface=args.interface,
|
||||
)
|
||||
if not iface:
|
||||
print(f"[WARN] 当前还没有接口拿到目标网段 {args.modem_subnet} 的地址")
|
||||
sys.exit(1)
|
||||
else:
|
||||
iface, ipv4_addrs = acquire_modem_interface(
|
||||
args.modem_subnet,
|
||||
explicit_iface=args.interface,
|
||||
)
|
||||
if not iface:
|
||||
print(f"[FAIL] 未找到落在目标网段 {args.modem_subnet} 内的模组接口")
|
||||
sys.exit(1)
|
||||
|
||||
print_interface_status(iface)
|
||||
|
||||
if ipv4_addrs:
|
||||
for addr in ipv4_addrs:
|
||||
print(f"[OK] {iface} 已获取 IPv4: {addr}")
|
||||
save_interface_info(iface)
|
||||
route_targets = dedupe_keep_order(args.route_target)
|
||||
if args.remove_default_route:
|
||||
enforce_route_policy(iface, args.gateway, route_targets)
|
||||
|
||||
connectivity_targets = route_targets or list(DEFAULT_PUBLIC_TARGETS)
|
||||
ping_via_interface(iface, targets=connectivity_targets)
|
||||
print(f"[DONE] RNDIS 拨号完成，可执行: sudo python3 speed_test.py {iface}")
|
||||
return
|
||||
|
||||
print(f"[WARN] {iface} 已出现，但还没有 IPv4 地址")
|
||||
print(f"[INFO] 可手动检查: ip addr show {iface}")
|
||||
sys.exit(1)
|
||||
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"[FAIL] {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
60
robot/v4l2/OmniSocketGo_robot/scripts/boot/robot-boot.env
Normal file
60
robot/v4l2/OmniSocketGo_robot/scripts/boot/robot-boot.env
Normal file
@@ -0,0 +1,60 @@
|
||||
# Boot-time settings for the robot-side autostart chain.
|
||||
# Override machine-specific values in robot-boot.env.local.
|
||||
|
||||
BLITZ_BOOT_DELAY_SEC="30"
|
||||
BLITZ_RUN_ROOT="/var/log/blitz-robot"
|
||||
BLITZ_LOG_FILE="/var/log/blitz-robot/startup.log"
|
||||
BLITZ_RUNTIME_DIR="/run/blitz-robot"
|
||||
BLITZ_RUN_CONTEXT_FILE="${BLITZ_RUNTIME_DIR}/run-context.env"
|
||||
BLITZ_RUN_ID_FILE="${BLITZ_RUNTIME_DIR}/run-id"
|
||||
BLITZ_CURRENT_RUN_LINK="${BLITZ_RUN_ROOT}/current"
|
||||
|
||||
BLITZ_5G_DIAL_DIR="${OMNISOCKETGO_ROOT}/scripts/boot"
|
||||
BLITZ_5G_SERIAL_PORT="/dev/ttyUSB2"
|
||||
BLITZ_5G_INTERFACE=""
|
||||
BLITZ_5G_MODEM_SUBNET="192.168.224.0/22"
|
||||
BLITZ_5G_GATEWAY="192.168.225.1"
|
||||
BLITZ_5G_SKIP_DHCP="0"
|
||||
BLITZ_5G_REMOVE_DEFAULT_ROUTE="1"
|
||||
BLITZ_5G_ROUTE_TARGETS="106.55.173.235"
|
||||
BLITZ_5G_INFO_JSON="${OMNISOCKETGO_ROOT}/scripts/boot/modem_network_info.json"
|
||||
BLITZ_5G_SERIAL_WAIT_SEC="60"
|
||||
BLITZ_5G_ROUTE_WAIT_SEC="30"
|
||||
|
||||
# Leave empty to fall back to the host part of ROBOT_SIDE_OMNISOCKET_SERVER_ADDR.
|
||||
BLITZ_TIME_SERVER_IP="81.70.156.140"
|
||||
|
||||
BLITZ_ROS_USER="nvidia"
|
||||
BLITZ_ROS_SOCKET_WAIT_SEC="20"
|
||||
BLITZ_WATCHDOG_INTERVAL_SEC="5"
|
||||
BLITZ_HEALTH_STALE_SEC="15"
|
||||
BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC="15"
|
||||
BLITZ_KCP_STATS_INTERVAL_MS="1000"
|
||||
BLITZ_CONTROL_LATENCY_LOG_ENABLED="1"
|
||||
BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD="100"
|
||||
BLITZ_CONTROL_ACK_SAMPLE_MOD="10"
|
||||
BLITZ_VIDEO_STAGE_LOG_ENABLED="1"
|
||||
BLITZ_VIDEO_STAGE_LOG_SAMPLE_MOD="10"
|
||||
BLITZ_5G_LINK_LOG_INTERVAL_SEC="5"
|
||||
BLITZ_JSONL_FLUSH_INTERVAL_MS="1000"
|
||||
BLITZ_JSONL_FLUSH_BYTES="262144"
|
||||
BLITZ_JSONL_ROTATE_BYTES="134217728"
|
||||
BLITZ_JSONL_ROTATE_FILES="8"
|
||||
# Log one normal relay packet out of every N packets. Drop events still log immediately.
|
||||
OMNI_RELAY_PACKET_LOG_SAMPLE_EVERY="200"
|
||||
BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC="5"
|
||||
BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC="30"
|
||||
BLITZ_NETWORK_FAIL_THRESHOLD="3"
|
||||
BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC="30"
|
||||
BLITZ_GPS_MONITOR_ENABLED="1"
|
||||
BLITZ_GPS_DEVICE_GLOB="/dev/ttyCH341USB*"
|
||||
BLITZ_GPS_CHECK_INTERVAL_SEC="10"
|
||||
BLITZ_GPS_RESTART_UNITS="gpsd.socket gpsd.service"
|
||||
BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION="0"
|
||||
|
||||
OMNI_CAMERA_DEVICE="/dev/v4l/by-path/platform-a80aa10000.usb-usb-0:3.2:1.4-video-index0"
|
||||
|
||||
# Boot units run b_side_omnid as root directly, so nested sudo must stay off.
|
||||
B_SIDE_OMNID_USE_SUDO="0"
|
||||
OMNI_CONTROL_ACK_PEER_ID="peer-b-ctrl-ack"
|
||||
OMNI_CONTROL_ACK_TARGET_PEER="peer-a-ctrl-ack"
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="5g-link-logger-service"
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_run_context
|
||||
|
||||
export OMNI_BOOT_MODE="1"
|
||||
export BLITZ_INSTANCE_ID="${BLITZ_INSTANCE_ID:-$(blitz_new_instance_id)}"
|
||||
export BLITZ_5G_LINK_LOG_PATH="${BLITZ_5G_LINK_LOG_PATH:-${BLITZ_RUN_DIR}/b-5g-link-quality.${BLITZ_INSTANCE_ID}.jsonl}"
|
||||
|
||||
blitz_log "${STEP}" "start" "start" "exec bash ${OMNISOCKETGO_ROOT}/scripts/boot/blitz-5g-link-logger.sh" 0
|
||||
exec bash "${OMNISOCKETGO_ROOT}/scripts/boot/blitz-5g-link-logger.sh"
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="b-side-omnid"
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_run_context
|
||||
|
||||
blitz_require_executable "${OMNISOCKETGO_ROOT}/bin/b_side_omnid" "${STEP}"
|
||||
|
||||
export OMNI_BOOT_MODE="1"
|
||||
export B_SIDE_OMNID_USE_SUDO="0"
|
||||
|
||||
blitz_log "${STEP}" "start" "start" "exec bash ${OMNISOCKETGO_ROOT}/scripts/dev/start-b-side-omnid.sh" 0
|
||||
exec bash "${OMNISOCKETGO_ROOT}/scripts/dev/start-b-side-omnid.sh"
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="ros-receiver"
|
||||
|
||||
blitz_load_boot_env
|
||||
blitz_require_run_context
|
||||
|
||||
blitz_require_file "/opt/ros/${ROS_DISTRO}/setup.bash" "${STEP}"
|
||||
blitz_require_file "${ROS_CONTROL_PY_DIR}/install/setup.bash" "${STEP}"
|
||||
|
||||
export OMNI_BOOT_MODE="1"
|
||||
blitz_log "${STEP}" "start" "start" "exec bash ${OMNISOCKETGO_ROOT}/scripts/dev/start-ros-receiver.sh" 0
|
||||
exec bash "${OMNISOCKETGO_ROOT}/scripts/dev/start-ros-receiver.sh"
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Blitz robot 5G dial
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-run-context.service
|
||||
Requires=blitz-run-context.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/5g-dial.sh
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Blitz robot 5G link logger
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-run-context.service blitz-5g-dial.service
|
||||
Requires=blitz-run-context.service
|
||||
Wants=blitz-run-context.service blitz-5g-dial.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=-/run/blitz-robot/run-context.env
|
||||
ExecStartPre=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/prepare-runtime-dir.sh
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/start-5g-link-logger-service.sh
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=Blitz robot b-side omnid
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-run-context.service blitz-5g-dial.service blitz-ros-receiver.service
|
||||
Requires=blitz-run-context.service
|
||||
Wants=blitz-run-context.service blitz-5g-dial.service blitz-ros-receiver.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=-/run/blitz-robot/run-context.env
|
||||
ExecStartPre=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/prepare-runtime-dir.sh
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/start-b-side-omnid-service.sh
|
||||
ExecStopPost=/bin/bash -lc 'if [[ "${SERVICE_RESULT:-success}" != "success" ]]; then exec /bin/bash "@OMNISOCKETGO_ROOT@/scripts/boot/blitz-incident-capture-launch.sh" --source exec-stop-post --unit "%n" --result "${SERVICE_RESULT:-}" --exit-status "${EXIT_STATUS:-}" --reason b-side-service-exit; fi'
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Blitz robot boot gate
|
||||
PartOf=blitz-robot.target
|
||||
After=multi-user.target network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/boot-gate.sh
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Blitz robot boot chain
|
||||
Wants=blitz-boot-gate.service
|
||||
Wants=blitz-run-context.service
|
||||
Wants=blitz-5g-dial.service
|
||||
Wants=blitz-5g-link-logger.service
|
||||
Wants=blitz-ros-receiver.service
|
||||
Wants=blitz-b-side-omnid.service
|
||||
Wants=blitz-watchdog.service
|
||||
After=multi-user.target
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Blitz robot ROS receiver
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-run-context.service blitz-5g-dial.service
|
||||
Requires=blitz-run-context.service
|
||||
Wants=blitz-run-context.service blitz-5g-dial.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=@BLITZ_ROS_USER@
|
||||
PermissionsStartOnly=true
|
||||
EnvironmentFile=-/run/blitz-robot/run-context.env
|
||||
ExecStartPre=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/prepare-runtime-dir.sh
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/start-ros-receiver-service.sh
|
||||
ExecStartPost=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/wait-for-unix-socket.sh --step ros-receiver
|
||||
ExecStopPost=/bin/bash -lc 'if [[ "${SERVICE_RESULT:-success}" != "success" ]]; then exec /bin/bash "@OMNISOCKETGO_ROOT@/scripts/boot/blitz-incident-capture-launch.sh" --source exec-stop-post --unit "%n" --result "${SERVICE_RESULT:-}" --exit-status "${EXIT_STATUS:-}" --reason ros-service-exit; fi'
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Blitz robot run context
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-boot-gate.service
|
||||
Requires=blitz-boot-gate.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/blitz-run-context.sh
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Blitz robot health watchdog
|
||||
PartOf=blitz-robot.target
|
||||
After=blitz-run-context.service blitz-b-side-omnid.service blitz-ros-receiver.service
|
||||
Requires=blitz-run-context.service
|
||||
Wants=blitz-run-context.service blitz-b-side-omnid.service blitz-ros-receiver.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=-/run/blitz-robot/run-context.env
|
||||
ExecStartPre=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/prepare-runtime-dir.sh
|
||||
ExecStart=/bin/bash @OMNISOCKETGO_ROOT@/scripts/boot/blitz-watchdog.sh
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:@BLITZ_LOG_FILE@
|
||||
StandardError=append:@BLITZ_LOG_FILE@
|
||||
|
||||
[Install]
|
||||
WantedBy=blitz-robot.target
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
STEP="ros-receiver"
|
||||
SOCKET_PATH=""
|
||||
TIMEOUT_SEC=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--path)
|
||||
SOCKET_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--timeout)
|
||||
TIMEOUT_SEC="$2"
|
||||
shift 2
|
||||
;;
|
||||
--step)
|
||||
STEP="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
blitz_log "${STEP}" "wait-socket-arg" "failure" "unknown argument: $1" 2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
blitz_load_boot_env
|
||||
|
||||
SOCKET_PATH="${SOCKET_PATH:-${ROBOT_RECEIVER_LOCAL_SOCKET_PATH}}"
|
||||
TIMEOUT_SEC="${TIMEOUT_SEC:-${BLITZ_ROS_SOCKET_WAIT_SEC}}"
|
||||
|
||||
blitz_log "${STEP}" "wait-socket" "start" "path=${SOCKET_PATH} timeout_sec=${TIMEOUT_SEC}" 0
|
||||
|
||||
for (( waited=0; waited< TIMEOUT_SEC; waited++ )); do
|
||||
if [[ -S "${SOCKET_PATH}" ]]; then
|
||||
blitz_log "${STEP}" "wait-socket" "success" "path=${SOCKET_PATH} waited_sec=${waited}" 0
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
blitz_log "${STEP}" "wait-socket" "failure" "path=${SOCKET_PATH} timeout_sec=${TIMEOUT_SEC}" 1
|
||||
exit 1
|
||||
188
robot/v4l2/OmniSocketGo_robot/scripts/dev/README.md
Normal file
188
robot/v4l2/OmniSocketGo_robot/scripts/dev/README.md
Normal file
@@ -0,0 +1,188 @@
|
||||
# Dev Startup Scripts
|
||||
|
||||
This directory lives inside the `OmniSocketGo` repo and acts as the main launch entry for the whole local setup.
|
||||
|
||||
Default layout:
|
||||
|
||||
```text
|
||||
~/Documents/
|
||||
OmniSocketGo/
|
||||
scripts/dev/
|
||||
robot-command-center/
|
||||
```
|
||||
|
||||
The scripts assume:
|
||||
|
||||
- `OmniSocketGo` is the current repo
|
||||
- `robot-command-center` is a sibling directory next to it
|
||||
|
||||
If your `robot-command-center` is elsewhere, set `ROBOT_COMMAND_CENTER_ROOT` in `robot-remote.env.local`.
|
||||
`start-backend.sh` and `start-frontend.sh` need that repo; `start-ros-receiver.sh` and `start-b-side-omnid.sh` do not.
|
||||
|
||||
## Files
|
||||
|
||||
- `robot-remote.env`: shared defaults for backend, frontend, ROS, and `b_side_omnid`
|
||||
- `robot-remote.env.local`: optional local override file loaded after `robot-remote.env`
|
||||
- `load-env.sh`: loads the shared environment into the current shell
|
||||
- `prepare-camera-device.sh`: reports V4L2 owners and optionally stops one explicitly configured camera service
|
||||
- `resolve-camera-device.sh`: resolves an RGB capture node by USB serial plus MJPEG resolution support
|
||||
- `apply-camera-controls.sh`: applies the camera preset before `b_side_omnid` starts
|
||||
- `start-backend.sh`: starts Django ASGI with `uvicorn`
|
||||
- `log-network-summary.py`: polls the backend `network/latest` API and appends compact JSONL snapshots
|
||||
- `start-frontend.sh`: starts the Vite dev server
|
||||
- `start-ros-receiver.sh`: starts the ROS2 `udp_teleop_bridge` receiver
|
||||
- `start-b-side-omnid.sh`: applies camera controls, then starts `./bin/b_side_omnid` and uses `sudo -E` by default
|
||||
- `start-dev-tmux.sh`: optional one-command `tmux` launcher for all four processes
|
||||
|
||||
## Usage
|
||||
|
||||
Run these from the `OmniSocketGo` repo root:
|
||||
|
||||
```bash
|
||||
bash scripts/dev/start-backend.sh
|
||||
bash scripts/dev/start-frontend.sh
|
||||
bash scripts/dev/start-ros-receiver.sh
|
||||
bash scripts/dev/start-b-side-omnid.sh
|
||||
```
|
||||
|
||||
If you prefer one command and use `tmux`:
|
||||
|
||||
```bash
|
||||
bash scripts/dev/start-dev-tmux.sh
|
||||
```
|
||||
|
||||
If you only want the shared environment for manual commands:
|
||||
|
||||
```bash
|
||||
source scripts/dev/load-env.sh
|
||||
```
|
||||
|
||||
When you launch via `start-*.sh`, you do not need to manually `export` the variables from
|
||||
`robot-remote.env` or `robot-remote.env.local`. `load-env.sh` loads those files with `set -a`,
|
||||
so the variables are exported automatically for the child process. Manual `export` is only needed
|
||||
if you bypass these scripts and start binaries directly from a clean shell.
|
||||
|
||||
## Customizing
|
||||
|
||||
Edit `scripts/dev/robot-remote.env` for shared changes such as:
|
||||
|
||||
- `ROBOT_COMMAND_CENTER_ROOT`
|
||||
- `CONTROL_SIDE_OMNISOCKET_SERVER_ADDR`
|
||||
- `CONTROL_SIDE_OMNISOCKET_RELAY_VIA`
|
||||
- `ROBOT_SIDE_OMNISOCKET_SERVER_ADDR`
|
||||
- `ROBOT_SIDE_OMNISOCKET_RELAY_VIA`
|
||||
- `VITE_API_BASE_URL`
|
||||
- `OMNI_CAMERA_DEVICE`
|
||||
- `OMNI_CAMERA_AUTO_DISCOVER=1` resolves both camera nodes on every start instead of trusting unstable `/dev/video*` numbers
|
||||
- `OMNI_CAMERA_HEAD_SERIAL` and `OMNI_CAMERA_WAIST_SERIAL` permanently map the physical cameras to the head/waist roles
|
||||
- `OMNI_CAMERA_HEAD_DEVICE` and `OMNI_CAMERA_WAIST_DEVICE` are fallback nodes when automatic discovery is disabled
|
||||
- `OMNI_CAMERA_DISCOVERY_WIDTH`, `OMNI_CAMERA_DISCOVERY_HEIGHT`, and `OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC` tune capability matching and retry time
|
||||
- `OMNI_CAMERA_ACTIVE=head|waist` selects the camera sent at startup
|
||||
|
||||
`b_side_omnid` keeps both configured cameras streaming and only decodes/encodes/sends the selected input. Send a text control message with body `camera:head` or `camera:waist` to `peer-b-ctrl` to switch without reopening either camera. After applying the selection, the robot replies to the sender with `{"type":"camera.selected","camera":"head|waist"}` so callers can confirm the actual state. The normal fixed-size binary robot control packets are unchanged.
|
||||
- `OMNI_CAMERA_OCCUPANCY_POLICY`
|
||||
- `OMNI_CAMERA_RELEASE_SERVICE`
|
||||
- `OMNI_CAMERA_PROFILE`
|
||||
- `OMNI_CAMERA_BRIGHTNESS`
|
||||
- `OMNI_CAMERA_CUSTOM_CTRL`
|
||||
- `OMNI_CAMERA_VERIFY`
|
||||
- `OMNI_VIDEO_PEER_ID`
|
||||
- `OMNI_CONTROL_PEER_ID`
|
||||
- `OMNI_VIDEO_SOFT_BACKPRESSURE_SEGMENTS`
|
||||
- `OMNI_VIDEO_HARD_BACKPRESSURE_SEGMENTS`
|
||||
- `OMNI_VIDEO_HARD_BACKPRESSURE_HOLD_MS`
|
||||
- `OMNI_CONTROL_SERVER_IDLE_RECONNECT_MS`
|
||||
- `OMNI_VIDEO_MAX_FRAME_AGE_MS`
|
||||
- `OMNISOCKET_TELEMETRY_PEER_ID`
|
||||
- `OMNISOCKET_TELEMETRY_INTERVAL_MS`
|
||||
- `OMNISOCKET_TELEMETRY_STALE_AFTER_MS`
|
||||
- `OMNI_NETWORK_SUMMARY_LOG_ENABLED`
|
||||
- `OMNI_NETWORK_SUMMARY_LOG_PATH`
|
||||
- `OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS`
|
||||
|
||||
Camera discovery uses `udevadm` and `v4l2-ctl` on the robot side. A candidate must have the configured serial number and advertise MJPEG at the configured capture resolution; depth, IR, Bayer, and metadata nodes are rejected. With `OMNI_CAMERA_OCCUPANCY_POLICY=release-known`, the start script stops the configured head/waist Orbbec services before discovery so libusb-owned interfaces can reattach to `uvcvideo`.
|
||||
|
||||
Role mapping:
|
||||
|
||||
- `start-backend.sh` uses the `CONTROL_SIDE_*` address pair
|
||||
- `start-b-side-omnid.sh` uses the `ROBOT_SIDE_*` address pair
|
||||
- `start-b-side-omnid.sh` also applies the `OMNI_CAMERA_*` preset before the daemon opens the camera
|
||||
- `start-b-side-omnid.sh` runs the camera occupancy preflight before applying camera controls
|
||||
- `start-ros-receiver.sh` defaults to the robot-side address pair, but with `transport=unix_dgram` it usually does not need the server address
|
||||
|
||||
New repair knobs:
|
||||
|
||||
- `OMNI_VIDEO_SOFT_BACKPRESSURE_SEGMENTS`, `OMNI_VIDEO_HARD_BACKPRESSURE_SEGMENTS`, and `OMNI_VIDEO_HARD_BACKPRESSURE_HOLD_MS` are used by `b_side_omnid`
|
||||
- `OMNI_CONTROL_SERVER_IDLE_RECONNECT_MS` is used by `b_side_omnid`
|
||||
- `OMNI_VIDEO_MAX_FRAME_AGE_MS` is used by `start-backend.sh` on the A-side backend, not by `b_side_omnid`
|
||||
- `OMNISOCKET_TELEMETRY_INTERVAL_MS` and `OMNISOCKET_TELEMETRY_STALE_AFTER_MS` tune the backend's D-side telemetry freshness window
|
||||
- `OMNI_NETWORK_SUMMARY_LOG_*` controls the A-side JSONL summary logger that polls `GET /api/network/latest/`
|
||||
|
||||
Default long-run network logging:
|
||||
|
||||
- A-side starts a compact JSONL logger by default at `${OMNISOCKETGO_ROOT}/logs/a-network-summary.jsonl`
|
||||
- The default A-side polling interval is `2000 ms`
|
||||
- For D-side long runs, prefer:
|
||||
|
||||
```bash
|
||||
./bin/kcpserver -listen 0.0.0.0:10909 \
|
||||
-telemetry-peer peer-a-telemetry \
|
||||
-telemetry-interval 1000ms \
|
||||
-kcp-session-stats-log logs/d-kcp-stats.jsonl \
|
||||
-kcp-session-stats-interval 1000ms
|
||||
```
|
||||
|
||||
- Keep `-latency-log` and `-kcp-ts-debug-log` off by default for multi-hour runs
|
||||
- Do not continuously redirect relay `C` stderr to a file unless you are reproducing a short issue window
|
||||
|
||||
Put machine-specific overrides into `scripts/dev/robot-remote.env.local`. Example:
|
||||
|
||||
```bash
|
||||
ROBOT_COMMAND_CENTER_ROOT="$HOME/Documents/robot-command-center"
|
||||
OMNI_CAMERA_DEVICE="/dev/video30"
|
||||
B_SIDE_OMNID_USE_SUDO="0"
|
||||
OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS="5000"
|
||||
```
|
||||
|
||||
Camera occupancy handling is deliberately narrow. `check` only reports owners and fails if the
|
||||
device is busy. `release-known` stops exactly `OMNI_CAMERA_RELEASE_SERVICE`, then checks the device
|
||||
again. It never kills an arbitrary PID and refuses to stop `proc_manager.service` automatically:
|
||||
|
||||
```bash
|
||||
OMNI_CAMERA_DEVICE="/dev/video18"
|
||||
OMNI_CAMERA_OCCUPANCY_POLICY="release-known"
|
||||
OMNI_CAMERA_RELEASE_SERVICE="orbbec_waist.service"
|
||||
```
|
||||
|
||||
Run the preflight without starting the daemon:
|
||||
|
||||
```bash
|
||||
bash scripts/dev/prepare-camera-device.sh
|
||||
```
|
||||
|
||||
If a remaining owner belongs to `proc_manager.service`, inspect it with `ros2 component list` and
|
||||
unload only the camera component. Stopping the complete process manager can interrupt unrelated
|
||||
robot functions.
|
||||
|
||||
Default camera behavior is the `night` preset:
|
||||
|
||||
```bash
|
||||
OMNI_CAMERA_PROFILE="night"
|
||||
# Optional per-machine tweak:
|
||||
OMNI_CAMERA_BRIGHTNESS="8"
|
||||
```
|
||||
|
||||
To switch to a daytime preset with brightness only:
|
||||
|
||||
```bash
|
||||
OMNI_CAMERA_PROFILE="day"
|
||||
OMNI_CAMERA_BRIGHTNESS="8"
|
||||
```
|
||||
|
||||
To send the raw `v4l2-ctl --set-ctrl=...` payload yourself:
|
||||
|
||||
```bash
|
||||
OMNI_CAMERA_PROFILE="custom"
|
||||
OMNI_CAMERA_CUSTOM_CTRL="brightness=8,auto_exposure=1,exposure_time_absolute=800,gain=64"
|
||||
OMNI_CAMERA_VERIFY="1"
|
||||
```
|
||||
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Aggregate run logs into control/video latency estimate outputs.")
|
||||
parser.add_argument("--run-dir", required=True, help="Run directory containing JSONL logs.")
|
||||
parser.add_argument("--output-dir", help="Output directory. Defaults to --run-dir.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def iter_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
if not path.exists():
|
||||
return records
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
records.append(payload)
|
||||
return records
|
||||
|
||||
|
||||
def load_glob_jsonl(run_dir: Path, pattern: str) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
for path in sorted(run_dir.glob(pattern)):
|
||||
records.extend(iter_jsonl(path))
|
||||
return records
|
||||
|
||||
|
||||
def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def parse_unix_ms(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.endswith("Z"):
|
||||
text = f"{text[:-1]}+00:00"
|
||||
try:
|
||||
return int(datetime.fromisoformat(text).astimezone(timezone.utc).timestamp() * 1000)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def flatten_net_epoch(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
flattened: list[dict[str, Any]] = []
|
||||
for sample in samples:
|
||||
links = sample.get("links") or {}
|
||||
a_to_d = (links.get("a_to_d") or {}).get("sessions") or {}
|
||||
d_to_b = (links.get("d_to_b") or {}).get("sessions") or {}
|
||||
a_control = (a_to_d.get("control") or {}).get("kcp") or {}
|
||||
d_control = (d_to_b.get("control") or {}).get("kcp") or {}
|
||||
a_video = (a_to_d.get("video") or {}).get("kcp") or {}
|
||||
d_video = (d_to_b.get("video") or {}).get("kcp") or {}
|
||||
flattened.append(
|
||||
{
|
||||
"updated_at": sample.get("updated_at"),
|
||||
"a_to_d_control_srtt_ms": a_control.get("srtt_ms"),
|
||||
"a_to_d_control_min_srtt_ms": a_control.get("min_srtt_ms"),
|
||||
"d_to_b_control_srtt_ms": d_control.get("srtt_ms"),
|
||||
"d_to_b_control_min_srtt_ms": d_control.get("min_srtt_ms"),
|
||||
"a_to_d_video_srtt_ms": a_video.get("srtt_ms"),
|
||||
"a_to_d_video_min_srtt_ms": a_video.get("min_srtt_ms"),
|
||||
"d_to_b_video_srtt_ms": d_video.get("srtt_ms"),
|
||||
"d_to_b_video_min_srtt_ms": d_video.get("min_srtt_ms"),
|
||||
"a_to_d_control_feedback_age_ms": a_control.get("last_feedback_age_ms"),
|
||||
"d_to_b_control_feedback_age_ms": d_control.get("last_feedback_age_ms"),
|
||||
"a_to_d_video_feedback_age_ms": a_video.get("last_feedback_age_ms"),
|
||||
"d_to_b_video_feedback_age_ms": d_video.get("last_feedback_age_ms"),
|
||||
"a_to_d_control_retrans_delta": ((a_to_d.get("control") or {}).get("trend") or {}).get("retrans_delta"),
|
||||
"d_to_b_control_retrans_delta": ((d_to_b.get("control") or {}).get("trend") or {}).get("retrans_delta"),
|
||||
"a_to_d_video_retrans_delta": ((a_to_d.get("video") or {}).get("trend") or {}).get("retrans_delta"),
|
||||
"d_to_b_video_retrans_delta": ((d_to_b.get("video") or {}).get("trend") or {}).get("retrans_delta"),
|
||||
"a_to_d_video_window_pressure_pct": a_video.get("window_pressure_pct"),
|
||||
"d_to_b_video_window_pressure_pct": d_video.get("window_pressure_pct"),
|
||||
"robot_health": sample.get("robot_health"),
|
||||
}
|
||||
)
|
||||
return flattened
|
||||
|
||||
|
||||
def aggregate_control_estimates(
|
||||
network_samples: list[dict[str, Any]],
|
||||
control_events: list[dict[str, Any]],
|
||||
control_acks: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
if control_acks:
|
||||
return control_acks
|
||||
|
||||
fallback: list[dict[str, Any]] = []
|
||||
for sample in network_samples:
|
||||
estimate = sample.get("latency_estimate") or {}
|
||||
fallback.append(
|
||||
{
|
||||
"updated_at": sample.get("updated_at"),
|
||||
"estimate_method": "srtt_fallback",
|
||||
"control_loop_rtt_ms": estimate.get("control_loop_rtt_ms"),
|
||||
"control_to_persist_est_ms": estimate.get("control_to_persist_est_ms"),
|
||||
"control_oneway_srtt_est_ms": estimate.get("control_oneway_srtt_est_ms"),
|
||||
"control_oneway_bestcase_est_ms": estimate.get("control_oneway_bestcase_est_ms"),
|
||||
"source_event_count": len(control_events),
|
||||
}
|
||||
)
|
||||
return fallback
|
||||
|
||||
|
||||
def aggregate_video_estimates(
|
||||
network_samples: list[dict[str, Any]],
|
||||
frame_recv_records: list[dict[str, Any]],
|
||||
display_probe_records: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
network_timeline = sorted(
|
||||
(
|
||||
(updated_at_ms, sample.get("latency_estimate") or {})
|
||||
for sample in network_samples
|
||||
for updated_at_ms in [parse_unix_ms(sample.get("updated_at"))]
|
||||
if updated_at_ms is not None
|
||||
),
|
||||
key=lambda item: item[0],
|
||||
)
|
||||
probes_by_seq = {
|
||||
int(record["frame_seq"]): record
|
||||
for record in display_probe_records
|
||||
if record.get("frame_seq") is not None
|
||||
}
|
||||
estimates: list[dict[str, Any]] = []
|
||||
timeline_index = 0
|
||||
|
||||
for record in frame_recv_records:
|
||||
frame_seq = record.get("frame_seq")
|
||||
if frame_seq is None:
|
||||
continue
|
||||
probe = probes_by_seq.get(int(frame_seq))
|
||||
backend_received_unix_ns = record.get("backend_received_unix_ns")
|
||||
backend_received_unix_ms = None
|
||||
try:
|
||||
if backend_received_unix_ns is not None:
|
||||
backend_received_unix_ms = int(int(backend_received_unix_ns) / 1_000_000)
|
||||
except (TypeError, ValueError):
|
||||
backend_received_unix_ms = None
|
||||
|
||||
latency_estimate: dict[str, Any] = {}
|
||||
if backend_received_unix_ms is not None and network_timeline:
|
||||
while timeline_index + 1 < len(network_timeline) and network_timeline[timeline_index + 1][0] <= backend_received_unix_ms:
|
||||
timeline_index += 1
|
||||
if network_timeline[timeline_index][0] <= backend_received_unix_ms:
|
||||
latency_estimate = network_timeline[timeline_index][1]
|
||||
|
||||
network_oneway = latency_estimate.get("video_network_oneway_est_ms")
|
||||
capture_to_send = record.get("b_side_capture_to_send_ms")
|
||||
partial_est = None
|
||||
if capture_to_send is not None or network_oneway is not None:
|
||||
partial_est = round(float(capture_to_send or 0.0) + float(network_oneway or 0.0), 3)
|
||||
request_to_paint_ms = None
|
||||
if probe is not None and probe.get("request_to_paint_ms") is not None:
|
||||
request_to_paint_ms = round(float(probe["request_to_paint_ms"]), 3)
|
||||
elif probe is not None and probe.get("request_started_unix_ms") is not None and probe.get("paint_unix_ms") is not None:
|
||||
request_to_paint_ms = round(float(probe["paint_unix_ms"]) - float(probe["request_started_unix_ms"]), 3)
|
||||
video_e2e_est_ms = round(partial_est + request_to_paint_ms, 3) if partial_est is not None and request_to_paint_ms is not None else None
|
||||
estimates.append(
|
||||
{
|
||||
"frame_seq": frame_seq,
|
||||
"backend_received_unix_ns": record.get("backend_received_unix_ns"),
|
||||
"frame_hash": record.get("frame_hash"),
|
||||
"estimate_method": "capture_to_send+srtt/2+request_to_paint" if video_e2e_est_ms is not None else "capture_to_send+srtt/2",
|
||||
"video_network_oneway_est_ms": network_oneway,
|
||||
"b_side_capture_to_send_ms": capture_to_send,
|
||||
"request_to_paint_ms": request_to_paint_ms,
|
||||
"response_to_paint_ms": probe.get("response_to_paint_ms") if probe is not None else None,
|
||||
"backend_to_request_ms": probe.get("backend_to_request_ms") if probe is not None else None,
|
||||
"backend_to_request_ms_raw": probe.get("backend_to_request_ms_raw") if probe is not None else None,
|
||||
"backend_to_paint_ms": probe.get("backend_to_paint_ms") if probe is not None else None,
|
||||
"backend_to_paint_ms_raw": probe.get("backend_to_paint_ms_raw") if probe is not None else None,
|
||||
"browser_backend_clock_offset_ms": probe.get("browser_backend_clock_offset_ms") if probe is not None else None,
|
||||
"browser_backend_clock_rtt_ms": probe.get("browser_backend_clock_rtt_ms") if probe is not None else None,
|
||||
"video_partial_est_ms": partial_est,
|
||||
"video_e2e_est_ms": video_e2e_est_ms,
|
||||
"sequence_gap": record.get("sequence_gap"),
|
||||
"repeat_flag": record.get("repeat_flag"),
|
||||
"sender_clock_delta_ms_raw": record.get("sender_clock_delta_ms_raw"),
|
||||
}
|
||||
)
|
||||
return estimates
|
||||
|
||||
|
||||
def write_html_summary(
|
||||
path: Path,
|
||||
*,
|
||||
net_epochs: list[dict[str, Any]],
|
||||
control_estimates: list[dict[str, Any]],
|
||||
video_estimates: list[dict[str, Any]],
|
||||
) -> None:
|
||||
latest_control = control_estimates[-1] if control_estimates else {}
|
||||
latest_video = video_estimates[-1] if video_estimates else {}
|
||||
latest_net = net_epochs[-1] if net_epochs else {}
|
||||
html_text = f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Latency Estimates</title>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; margin: 24px; background: #0b1020; color: #eef2ff; }}
|
||||
.grid {{ display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; }}
|
||||
.card {{ border: 1px solid #334155; border-radius: 8px; padding: 16px; background: #111827; }}
|
||||
h1, h2 {{ margin-top: 0; }}
|
||||
p {{ margin: 6px 0; line-height: 1.5; }}
|
||||
code {{ color: #93c5fd; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Latency Estimates</h1>
|
||||
<div class="grid">
|
||||
<section class="card">
|
||||
<h2>Control</h2>
|
||||
<p><strong>loop RTT:</strong> {html.escape(str(latest_control.get("control_loop_rtt_ms")))}</p>
|
||||
<p><strong>to persist:</strong> {html.escape(str(latest_control.get("control_to_persist_est_ms")))}</p>
|
||||
<p><strong>method:</strong> {html.escape(str(latest_control.get("estimate_method")))}</p>
|
||||
<p><strong>samples:</strong> {len(control_estimates)}</p>
|
||||
</section>
|
||||
<section class="card">
|
||||
<h2>Video</h2>
|
||||
<p><strong>network one-way:</strong> {html.escape(str(latest_video.get("video_network_oneway_est_ms")))}</p>
|
||||
<p><strong>partial:</strong> {html.escape(str(latest_video.get("video_partial_est_ms")))}</p>
|
||||
<p><strong>end-to-end:</strong> {html.escape(str(latest_video.get("video_e2e_est_ms")))}</p>
|
||||
<p><strong>samples:</strong> {len(video_estimates)}</p>
|
||||
</section>
|
||||
<section class="card">
|
||||
<h2>Net Epoch</h2>
|
||||
<p><strong>a→d control srtt:</strong> {html.escape(str(latest_net.get("a_to_d_control_srtt_ms")))}</p>
|
||||
<p><strong>d→b control srtt:</strong> {html.escape(str(latest_net.get("d_to_b_control_srtt_ms")))}</p>
|
||||
<p><strong>a→d video srtt:</strong> {html.escape(str(latest_net.get("a_to_d_video_srtt_ms")))}</p>
|
||||
<p><strong>d→b video srtt:</strong> {html.escape(str(latest_net.get("d_to_b_video_srtt_ms")))}</p>
|
||||
</section>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
path.write_text(html_text, encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
run_dir = Path(args.run_dir).resolve()
|
||||
output_dir = Path(args.output_dir).resolve() if args.output_dir else run_dir
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
network_samples = load_glob_jsonl(run_dir, "a-network-summary.*.jsonl")
|
||||
control_events = load_glob_jsonl(run_dir, "a-control-events.*.jsonl")
|
||||
control_acks = load_glob_jsonl(run_dir, "a-control-acks.*.jsonl")
|
||||
frame_recv_records = load_glob_jsonl(run_dir, "a-video-frame-recv.*.jsonl")
|
||||
display_probe_records = load_glob_jsonl(run_dir, "a-video-display-probe.*.jsonl")
|
||||
|
||||
net_epochs = flatten_net_epoch(network_samples)
|
||||
control_estimates = aggregate_control_estimates(network_samples, control_events, control_acks)
|
||||
video_estimates = aggregate_video_estimates(network_samples, frame_recv_records, display_probe_records)
|
||||
|
||||
write_jsonl(output_dir / "net-epoch-summary.jsonl", net_epochs)
|
||||
write_jsonl(output_dir / "control-latency-estimates.jsonl", control_estimates)
|
||||
write_jsonl(output_dir / "video-latency-estimates.jsonl", video_estimates)
|
||||
write_html_summary(
|
||||
output_dir / "latency-estimates.html",
|
||||
net_epochs=net_epochs,
|
||||
control_estimates=control_estimates,
|
||||
video_estimates=video_estimates,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
|
||||
camera_device="${OMNI_CAMERA_DEVICE}"
|
||||
camera_profile="${OMNI_CAMERA_PROFILE}"
|
||||
camera_brightness="${OMNI_CAMERA_BRIGHTNESS}"
|
||||
camera_custom_ctrl="${OMNI_CAMERA_CUSTOM_CTRL}"
|
||||
camera_verify="${OMNI_CAMERA_VERIFY}"
|
||||
|
||||
is_truthy() {
|
||||
case "${1:-0}" in
|
||||
1|true|TRUE|yes|YES|on|ON)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
require_v4l2_ctl() {
|
||||
if command -v v4l2-ctl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Missing required command: v4l2-ctl. Install v4l-utils on the robot side before starting b_side_omnid." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
run_v4l2_ctl() {
|
||||
v4l2-ctl -d "${camera_device}" "$@"
|
||||
}
|
||||
|
||||
set_ctrl() {
|
||||
local ctrl="$1"
|
||||
|
||||
echo "[camera-controls] set ${camera_device} ${ctrl}"
|
||||
run_v4l2_ctl "--set-ctrl=${ctrl}"
|
||||
}
|
||||
|
||||
verify_ctrl() {
|
||||
local ctrl="$1"
|
||||
|
||||
echo "[camera-controls] verify ${camera_device} ${ctrl}"
|
||||
run_v4l2_ctl "--get-ctrl=${ctrl}"
|
||||
}
|
||||
|
||||
needs_v4l2_ctl=0
|
||||
|
||||
case "${camera_profile}" in
|
||||
night)
|
||||
needs_v4l2_ctl=1
|
||||
;;
|
||||
day)
|
||||
if [[ -n "${camera_brightness}" ]]; then
|
||||
needs_v4l2_ctl=1
|
||||
fi
|
||||
;;
|
||||
custom)
|
||||
if [[ -z "${camera_custom_ctrl}" ]]; then
|
||||
echo "OMNI_CAMERA_CUSTOM_CTRL must be non-empty when OMNI_CAMERA_PROFILE=custom." >&2
|
||||
exit 1
|
||||
fi
|
||||
needs_v4l2_ctl=1
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported OMNI_CAMERA_PROFILE: ${camera_profile}. Expected one of: night, day, custom." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if is_truthy "${camera_verify}"; then
|
||||
needs_v4l2_ctl=1
|
||||
fi
|
||||
|
||||
if [[ "${needs_v4l2_ctl}" == "0" ]]; then
|
||||
echo "[camera-controls] profile=${camera_profile}; no camera controls requested"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_v4l2_ctl
|
||||
|
||||
case "${camera_profile}" in
|
||||
night)
|
||||
set_ctrl "auto_exposure=1"
|
||||
set_ctrl "exposure_time_absolute=800"
|
||||
set_ctrl "gain=64"
|
||||
if [[ -n "${camera_brightness}" ]]; then
|
||||
set_ctrl "brightness=${camera_brightness}"
|
||||
fi
|
||||
;;
|
||||
day)
|
||||
if [[ -n "${camera_brightness}" ]]; then
|
||||
set_ctrl "brightness=${camera_brightness}"
|
||||
fi
|
||||
;;
|
||||
custom)
|
||||
set_ctrl "${camera_custom_ctrl}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if is_truthy "${camera_verify}"; then
|
||||
verify_ctrl "auto_exposure"
|
||||
verify_ctrl "exposure_time_absolute"
|
||||
verify_ctrl "gain"
|
||||
verify_ctrl "brightness"
|
||||
fi
|
||||
335
robot/v4l2/OmniSocketGo_robot/scripts/dev/load-env.sh
Normal file
335
robot/v4l2/OmniSocketGo_robot/scripts/dev/load-env.sh
Normal file
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
LOAD_ENV_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEFAULT_OMNISOCKETGO_ROOT="$(cd "${LOAD_ENV_SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
die() {
|
||||
echo "$*" >&2
|
||||
return 1 2>/dev/null || exit 1
|
||||
}
|
||||
|
||||
normalize_loaded_env_vars() {
|
||||
local var_name
|
||||
local value
|
||||
|
||||
for var_name in $(compgen -A variable); do
|
||||
case "${var_name}" in
|
||||
BACKEND_*|BLITZ_*|B_SIDE_*|CONTROL_*|FRONTEND_*|OMNI_*|PYTHON3_BIN|PYTHON_VENV_PATH|ROBOT_*|ROS_DISTRO|VITE_*)
|
||||
value="${!var_name}"
|
||||
if [[ "${value}" == *$'\r' ]]; then
|
||||
printf -v "${var_name}" '%s' "${value%$'\r'}"
|
||||
export "${var_name}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
is_omnisocketgo_root() {
|
||||
local dir="$1"
|
||||
[[ -f "${dir}/Makefile" && -f "${dir}/cmd/b_side_omnid.c" && -d "${dir}/ros-control-py" ]]
|
||||
}
|
||||
|
||||
is_robot_command_center_root() {
|
||||
local dir="$1"
|
||||
[[ -f "${dir}/backend/config/asgi.py" && -f "${dir}/frontend/package.json" ]]
|
||||
}
|
||||
|
||||
require_robot_command_center_root() {
|
||||
if ! is_robot_command_center_root "${ROBOT_COMMAND_CENTER_ROOT}"; then
|
||||
die "ROBOT_COMMAND_CENTER_ROOT must point to the robot-command-center repo root. Current value: ${ROBOT_COMMAND_CENTER_ROOT}. Set it in ${LOAD_ENV_SCRIPT_DIR}/robot-remote.env.local if needed."
|
||||
fi
|
||||
}
|
||||
|
||||
export OMNISOCKETGO_ROOT="${OMNISOCKETGO_ROOT:-${DEFAULT_OMNISOCKETGO_ROOT}}"
|
||||
|
||||
omni_camera_device_was_set=0
|
||||
omni_camera_profile_was_set=0
|
||||
omni_camera_brightness_was_set=0
|
||||
omni_camera_custom_ctrl_was_set=0
|
||||
omni_camera_verify_was_set=0
|
||||
|
||||
if [[ "${OMNI_CAMERA_DEVICE+x}" == "x" ]]; then
|
||||
omni_camera_device_was_set=1
|
||||
preserved_omni_camera_device="${OMNI_CAMERA_DEVICE}"
|
||||
fi
|
||||
if [[ "${OMNI_CAMERA_PROFILE+x}" == "x" ]]; then
|
||||
omni_camera_profile_was_set=1
|
||||
preserved_omni_camera_profile="${OMNI_CAMERA_PROFILE}"
|
||||
fi
|
||||
if [[ "${OMNI_CAMERA_BRIGHTNESS+x}" == "x" ]]; then
|
||||
omni_camera_brightness_was_set=1
|
||||
preserved_omni_camera_brightness="${OMNI_CAMERA_BRIGHTNESS}"
|
||||
fi
|
||||
if [[ "${OMNI_CAMERA_CUSTOM_CTRL+x}" == "x" ]]; then
|
||||
omni_camera_custom_ctrl_was_set=1
|
||||
preserved_omni_camera_custom_ctrl="${OMNI_CAMERA_CUSTOM_CTRL}"
|
||||
fi
|
||||
if [[ "${OMNI_CAMERA_VERIFY+x}" == "x" ]]; then
|
||||
omni_camera_verify_was_set=1
|
||||
preserved_omni_camera_verify="${OMNI_CAMERA_VERIFY}"
|
||||
fi
|
||||
|
||||
ENV_FILES=(
|
||||
"${LOAD_ENV_SCRIPT_DIR}/robot-remote.env"
|
||||
"${LOAD_ENV_SCRIPT_DIR}/robot-remote.env.local"
|
||||
)
|
||||
|
||||
for env_file in "${ENV_FILES[@]}"; do
|
||||
if [[ -f "${env_file}" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${env_file}"
|
||||
set +a
|
||||
fi
|
||||
done
|
||||
|
||||
normalize_loaded_env_vars
|
||||
|
||||
if [[ "${omni_camera_device_was_set}" == "1" ]]; then
|
||||
export OMNI_CAMERA_DEVICE="${preserved_omni_camera_device}"
|
||||
fi
|
||||
if [[ "${omni_camera_profile_was_set}" == "1" ]]; then
|
||||
export OMNI_CAMERA_PROFILE="${preserved_omni_camera_profile}"
|
||||
fi
|
||||
if [[ "${omni_camera_brightness_was_set}" == "1" ]]; then
|
||||
export OMNI_CAMERA_BRIGHTNESS="${preserved_omni_camera_brightness}"
|
||||
fi
|
||||
if [[ "${omni_camera_custom_ctrl_was_set}" == "1" ]]; then
|
||||
export OMNI_CAMERA_CUSTOM_CTRL="${preserved_omni_camera_custom_ctrl}"
|
||||
fi
|
||||
if [[ "${omni_camera_verify_was_set}" == "1" ]]; then
|
||||
export OMNI_CAMERA_VERIFY="${preserved_omni_camera_verify}"
|
||||
fi
|
||||
|
||||
export OMNISOCKETGO_ROOT="${OMNISOCKETGO_ROOT:-${DEFAULT_OMNISOCKETGO_ROOT}}"
|
||||
export ROBOT_COMMAND_CENTER_ROOT="${ROBOT_COMMAND_CENTER_ROOT:-$(dirname "${OMNISOCKETGO_ROOT}")/robot-command-center}"
|
||||
|
||||
if ! is_omnisocketgo_root "${OMNISOCKETGO_ROOT}"; then
|
||||
die "OMNISOCKETGO_ROOT must point to the OmniSocketGo repo root. Current value: ${OMNISOCKETGO_ROOT}"
|
||||
fi
|
||||
|
||||
export BACKEND_DIR="${BACKEND_DIR:-${ROBOT_COMMAND_CENTER_ROOT}/backend}"
|
||||
export FRONTEND_DIR="${FRONTEND_DIR:-${ROBOT_COMMAND_CENTER_ROOT}/frontend}"
|
||||
export ROS_CONTROL_PY_DIR="${ROS_CONTROL_PY_DIR:-${OMNISOCKETGO_ROOT}/ros-control-py}"
|
||||
export PYTHON3_BIN="${PYTHON3_BIN:-python3}"
|
||||
export PYTHON_VENV_PATH="${PYTHON_VENV_PATH:-${OMNISOCKETGO_ROOT}/.venv}"
|
||||
export BACKEND_HOST="${BACKEND_HOST:-0.0.0.0}"
|
||||
export BACKEND_PORT="${BACKEND_PORT:-8001}"
|
||||
export FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"
|
||||
export FRONTEND_PORT="${FRONTEND_PORT:-5173}"
|
||||
export OMNISOCKET_TELEMETRY_PEER_ID="${OMNISOCKET_TELEMETRY_PEER_ID:-peer-a-telemetry}"
|
||||
export OMNISOCKET_TELEMETRY_INTERVAL_MS="${OMNISOCKET_TELEMETRY_INTERVAL_MS:-1000}"
|
||||
export OMNISOCKET_TELEMETRY_STALE_AFTER_MS="${OMNISOCKET_TELEMETRY_STALE_AFTER_MS:-3000}"
|
||||
export OMNI_NETWORK_SUMMARY_LOG_ENABLED="${OMNI_NETWORK_SUMMARY_LOG_ENABLED:-1}"
|
||||
export OMNI_NETWORK_SUMMARY_LOG_PATH="${OMNI_NETWORK_SUMMARY_LOG_PATH:-${OMNISOCKETGO_ROOT}/logs/a-network-summary.jsonl}"
|
||||
export OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS="${OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS:-1000}"
|
||||
export OMNI_NETWORK_SUMMARY_LOG_REQUEST_TIMEOUT_SEC="${OMNI_NETWORK_SUMMARY_LOG_REQUEST_TIMEOUT_SEC:-3}"
|
||||
export CONTROL_SIDE_OMNISOCKET_SERVER_ADDR="${CONTROL_SIDE_OMNISOCKET_SERVER_ADDR:-}"
|
||||
export CONTROL_SIDE_OMNISOCKET_RELAY_VIA="${CONTROL_SIDE_OMNISOCKET_RELAY_VIA:-}"
|
||||
export ROBOT_SIDE_OMNISOCKET_SERVER_ADDR="${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}"
|
||||
export ROBOT_SIDE_OMNISOCKET_RELAY_VIA="${ROBOT_SIDE_OMNISOCKET_RELAY_VIA:-}"
|
||||
export ROS_DISTRO="${ROS_DISTRO:-jazzy}"
|
||||
export ROBOT_RECEIVER_TRANSPORT="${ROBOT_RECEIVER_TRANSPORT:-unix_dgram}"
|
||||
export ROBOT_RECEIVER_SERVER_ADDR="${ROBOT_RECEIVER_SERVER_ADDR:-${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}}"
|
||||
export ROBOT_RECEIVER_RELAY_VIA="${ROBOT_RECEIVER_RELAY_VIA:-${ROBOT_SIDE_OMNISOCKET_RELAY_VIA:-}}"
|
||||
export ROBOT_RECEIVER_PEER_ID="${ROBOT_RECEIVER_PEER_ID:-ros-bridge-ctrl}"
|
||||
export ROBOT_RECEIVER_EXPECTED_SENDER="${ROBOT_RECEIVER_EXPECTED_SENDER:-}"
|
||||
export ROBOT_RECEIVER_LOCAL_SOCKET_PATH="${ROBOT_RECEIVER_LOCAL_SOCKET_PATH:-/tmp/omnisocket-b-side-cmd.sock}"
|
||||
export ROBOT_RECEIVER_OUTPUT_TOPIC="${ROBOT_RECEIVER_OUTPUT_TOPIC:-/hric/robot/cmd_vel}"
|
||||
export ROBOT_RECEIVER_FRAME_ID="${ROBOT_RECEIVER_FRAME_ID:-pelvis}"
|
||||
export ROBOT_RECEIVER_WATCHDOG_TIMEOUT="${ROBOT_RECEIVER_WATCHDOG_TIMEOUT:-0.5}"
|
||||
export ROBOT_RECEIVER_PUBLISH_RATE_HZ="${ROBOT_RECEIVER_PUBLISH_RATE_HZ:-100.0}"
|
||||
export OMNI_CAMERA_DEVICE="${OMNI_CAMERA_DEVICE:-/dev/video0}"
|
||||
export OMNI_CAMERA_HEAD_DEVICE="${OMNI_CAMERA_HEAD_DEVICE:-/dev/video26}"
|
||||
export OMNI_CAMERA_WAIST_DEVICE="${OMNI_CAMERA_WAIST_DEVICE:-/dev/video18}"
|
||||
export OMNI_CAMERA_AUTO_DISCOVER="${OMNI_CAMERA_AUTO_DISCOVER:-0}"
|
||||
export OMNI_CAMERA_HEAD_SERIAL="${OMNI_CAMERA_HEAD_SERIAL:-}"
|
||||
export OMNI_CAMERA_WAIST_SERIAL="${OMNI_CAMERA_WAIST_SERIAL:-}"
|
||||
export OMNI_CAMERA_DISCOVERY_WIDTH="${OMNI_CAMERA_DISCOVERY_WIDTH:-1280}"
|
||||
export OMNI_CAMERA_DISCOVERY_HEIGHT="${OMNI_CAMERA_DISCOVERY_HEIGHT:-720}"
|
||||
export OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC="${OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC:-10}"
|
||||
export OMNI_CAMERA_HEAD_RELEASE_SERVICE="${OMNI_CAMERA_HEAD_RELEASE_SERVICE:-orbbec_head.service}"
|
||||
export OMNI_CAMERA_WAIST_RELEASE_SERVICE="${OMNI_CAMERA_WAIST_RELEASE_SERVICE:-orbbec_waist.service}"
|
||||
export OMNI_CAMERA_ACTIVE="${OMNI_CAMERA_ACTIVE:-head}"
|
||||
export OMNI_CAMERA_PROFILE="${OMNI_CAMERA_PROFILE:-night}"
|
||||
export OMNI_CAMERA_BRIGHTNESS="${OMNI_CAMERA_BRIGHTNESS:-}"
|
||||
export OMNI_CAMERA_CUSTOM_CTRL="${OMNI_CAMERA_CUSTOM_CTRL:-}"
|
||||
export OMNI_CAMERA_VERIFY="${OMNI_CAMERA_VERIFY:-0}"
|
||||
export OMNI_GPSD_HOST="${OMNI_GPSD_HOST:-127.0.0.1}"
|
||||
export OMNI_VIDEO_SERVER_ADDR="${OMNI_VIDEO_SERVER_ADDR:-${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}}"
|
||||
export OMNI_VIDEO_RELAY_VIA="${OMNI_VIDEO_RELAY_VIA:-${ROBOT_SIDE_OMNISOCKET_RELAY_VIA:-}}"
|
||||
export OMNI_CONTROL_SERVER_ADDR="${OMNI_CONTROL_SERVER_ADDR:-${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}}"
|
||||
export OMNI_CONTROL_RELAY_VIA="${OMNI_CONTROL_RELAY_VIA:-${ROBOT_SIDE_OMNISOCKET_RELAY_VIA:-}}"
|
||||
export OMNI_CONTROL_UNIX_SOCKET_PATH="${OMNI_CONTROL_UNIX_SOCKET_PATH:-${ROBOT_RECEIVER_LOCAL_SOCKET_PATH}}"
|
||||
export OMNI_CONTROL_ACK_PEER_ID="${OMNI_CONTROL_ACK_PEER_ID:-peer-b-ctrl-ack}"
|
||||
export OMNI_CONTROL_ACK_TARGET_PEER="${OMNI_CONTROL_ACK_TARGET_PEER:-peer-a-ctrl-ack}"
|
||||
export B_SIDE_OMNID_USE_SUDO="${B_SIDE_OMNID_USE_SUDO:-1}"
|
||||
export BLITZ_RUNTIME_DIR="${BLITZ_RUNTIME_DIR:-${OMNISOCKETGO_ROOT}/logs/runtime}"
|
||||
export BLITZ_RUN_ROOT="${BLITZ_RUN_ROOT:-${OMNISOCKETGO_ROOT}/logs}"
|
||||
export BLITZ_RUN_CONTEXT_FILE="${BLITZ_RUN_CONTEXT_FILE:-${BLITZ_RUNTIME_DIR}/run-context.env}"
|
||||
export BLITZ_RUN_ID_FILE="${BLITZ_RUN_ID_FILE:-${BLITZ_RUNTIME_DIR}/run-id}"
|
||||
export BLITZ_CURRENT_RUN_LINK="${BLITZ_CURRENT_RUN_LINK:-${BLITZ_RUN_ROOT}/current}"
|
||||
export BLITZ_5G_INTERFACE="${BLITZ_5G_INTERFACE:-}"
|
||||
export BLITZ_5G_MODEM_SUBNET="${BLITZ_5G_MODEM_SUBNET:-192.168.224.0/22}"
|
||||
export BLITZ_5G_GATEWAY="${BLITZ_5G_GATEWAY:-192.168.225.1}"
|
||||
export BLITZ_5G_ROUTE_TARGETS="${BLITZ_5G_ROUTE_TARGETS:-106.55.173.235}"
|
||||
export BLITZ_5G_INFO_JSON="${BLITZ_5G_INFO_JSON:-${OMNISOCKETGO_ROOT}/scripts/boot/modem_network_info.json}"
|
||||
export BLITZ_TIME_SERVER_IP="${BLITZ_TIME_SERVER_IP:-}"
|
||||
export BLITZ_KCP_STATS_INTERVAL_MS="${BLITZ_KCP_STATS_INTERVAL_MS:-1000}"
|
||||
export BLITZ_CONTROL_LATENCY_LOG_ENABLED="${BLITZ_CONTROL_LATENCY_LOG_ENABLED:-1}"
|
||||
export BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD="${BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD:-100}"
|
||||
export BLITZ_CONTROL_ACK_SAMPLE_MOD="${BLITZ_CONTROL_ACK_SAMPLE_MOD:-10}"
|
||||
export BLITZ_VIDEO_STAGE_LOG_ENABLED="${BLITZ_VIDEO_STAGE_LOG_ENABLED:-1}"
|
||||
export BLITZ_VIDEO_STAGE_LOG_SAMPLE_MOD="${BLITZ_VIDEO_STAGE_LOG_SAMPLE_MOD:-10}"
|
||||
export BLITZ_5G_LINK_LOG_INTERVAL_SEC="${BLITZ_5G_LINK_LOG_INTERVAL_SEC:-5}"
|
||||
export BLITZ_JSONL_FLUSH_INTERVAL_MS="${BLITZ_JSONL_FLUSH_INTERVAL_MS:-1000}"
|
||||
export BLITZ_JSONL_FLUSH_BYTES="${BLITZ_JSONL_FLUSH_BYTES:-262144}"
|
||||
export BLITZ_JSONL_ROTATE_BYTES="${BLITZ_JSONL_ROTATE_BYTES:-134217728}"
|
||||
export BLITZ_JSONL_ROTATE_FILES="${BLITZ_JSONL_ROTATE_FILES:-8}"
|
||||
|
||||
blitz_dev_utc_compact_timestamp() {
|
||||
date -u '+%Y%m%dT%H%M%SZ'
|
||||
}
|
||||
|
||||
blitz_dev_git_commit() {
|
||||
git -C "${OMNISOCKETGO_ROOT}" rev-parse HEAD 2>/dev/null || true
|
||||
}
|
||||
|
||||
blitz_dev_git_dirty_flag() {
|
||||
if git -C "${OMNISOCKETGO_ROOT}" diff --quiet --ignore-submodules=dirty >/dev/null 2>&1; then
|
||||
printf '0\n'
|
||||
return 0
|
||||
fi
|
||||
printf '1\n'
|
||||
}
|
||||
|
||||
blitz_dev_prepare_dirs() {
|
||||
mkdir -p "${BLITZ_RUNTIME_DIR}" "${BLITZ_RUN_ROOT}/runs" "${BLITZ_RUN_ROOT}/incidents"
|
||||
}
|
||||
|
||||
blitz_dev_write_run_info() {
|
||||
local run_dir="$1"
|
||||
local run_id="$2"
|
||||
local boot_id="$3"
|
||||
local tmp_info="${run_dir}/run-info.json.tmp.$$"
|
||||
local started_at
|
||||
local commit_hash
|
||||
local dirty_flag
|
||||
|
||||
started_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
commit_hash="$(blitz_dev_git_commit)"
|
||||
dirty_flag="$(blitz_dev_git_dirty_flag)"
|
||||
|
||||
python3 - "${tmp_info}" "${run_id}" "${run_dir}" "${boot_id}" "${started_at}" "${commit_hash}" "${dirty_flag}" "${HOSTNAME:-$(hostname)}" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
path, run_id, run_dir, boot_id, started_at, commit_hash, dirty_flag, hostname = sys.argv[1:9]
|
||||
payload = {
|
||||
"run_id": run_id,
|
||||
"run_dir": run_dir,
|
||||
"boot_id": boot_id,
|
||||
"started_at": started_at,
|
||||
"hostname": hostname,
|
||||
"git_commit": commit_hash,
|
||||
"git_dirty": dirty_flag == "1",
|
||||
"env": {
|
||||
key: os.environ.get(key, "")
|
||||
for key in sorted(os.environ)
|
||||
if key.startswith(("BLITZ_", "OMNI_", "ROBOT_RECEIVER_"))
|
||||
},
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
PY
|
||||
mv -f "${tmp_info}" "${run_dir}/run-info.json"
|
||||
}
|
||||
|
||||
blitz_dev_init_run_context() {
|
||||
local run_id="${1:-$(blitz_dev_utc_compact_timestamp)}"
|
||||
local boot_id="dev-$(blitz_dev_utc_compact_timestamp)"
|
||||
local run_dir="${BLITZ_RUN_ROOT}/runs/${run_id}"
|
||||
local tmp_context="${BLITZ_RUN_CONTEXT_FILE}.tmp.$$"
|
||||
|
||||
blitz_dev_prepare_dirs
|
||||
mkdir -p "${run_dir}"
|
||||
export BLITZ_RUN_ID="${run_id}"
|
||||
export BLITZ_RUN_DIR="${run_dir}"
|
||||
export BLITZ_BOOT_ID="${boot_id}"
|
||||
printf '%s\n' "${run_id}" > "${BLITZ_RUN_ID_FILE}"
|
||||
cat > "${tmp_context}" <<EOF
|
||||
BLITZ_RUN_ID=${run_id}
|
||||
BLITZ_RUN_DIR=${run_dir}
|
||||
BLITZ_BOOT_ID=${boot_id}
|
||||
BLITZ_RUN_ROOT=${BLITZ_RUN_ROOT}
|
||||
EOF
|
||||
mv -f "${tmp_context}" "${BLITZ_RUN_CONTEXT_FILE}"
|
||||
ln -sfn "${run_dir}" "${BLITZ_CURRENT_RUN_LINK}"
|
||||
blitz_dev_write_run_info "${run_dir}" "${run_id}" "${boot_id}"
|
||||
}
|
||||
|
||||
blitz_dev_load_run_context() {
|
||||
if [[ ! -f "${BLITZ_RUN_CONTEXT_FILE}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${BLITZ_RUN_CONTEXT_FILE}"
|
||||
set +a
|
||||
}
|
||||
|
||||
blitz_dev_ensure_run_context() {
|
||||
if blitz_dev_load_run_context; then
|
||||
return 0
|
||||
fi
|
||||
blitz_dev_init_run_context
|
||||
}
|
||||
|
||||
blitz_dev_reset_run_context() {
|
||||
rm -f "${BLITZ_RUN_CONTEXT_FILE}" "${BLITZ_RUN_ID_FILE}"
|
||||
blitz_dev_init_run_context
|
||||
}
|
||||
|
||||
blitz_dev_init_instance_context() {
|
||||
if [[ -z "${BLITZ_INSTANCE_ID:-}" ]]; then
|
||||
export BLITZ_INSTANCE_ID="$(blitz_dev_utc_compact_timestamp)-$$"
|
||||
fi
|
||||
}
|
||||
|
||||
blitz_dev_component_log_path() {
|
||||
local stem="$1"
|
||||
printf '%s/%s.%s.jsonl\n' "${BLITZ_RUN_DIR}" "${stem}" "${BLITZ_INSTANCE_ID}"
|
||||
}
|
||||
|
||||
blitz_dev_prepare_backend_logging_env() {
|
||||
blitz_dev_init_instance_context
|
||||
if [[ "${OMNI_NETWORK_SUMMARY_LOG_PATH}" == "${OMNISOCKETGO_ROOT}/logs/a-network-summary.jsonl" ]]; then
|
||||
export OMNI_NETWORK_SUMMARY_LOG_PATH
|
||||
OMNI_NETWORK_SUMMARY_LOG_PATH="$(blitz_dev_component_log_path "a-network-summary")"
|
||||
fi
|
||||
export BLITZ_A_CONTROL_EVENTS_LOG_PATH="${BLITZ_A_CONTROL_EVENTS_LOG_PATH:-$(blitz_dev_component_log_path "a-control-events")}"
|
||||
export BLITZ_A_CONTROL_ACKS_LOG_PATH="${BLITZ_A_CONTROL_ACKS_LOG_PATH:-$(blitz_dev_component_log_path "a-control-acks")}"
|
||||
export BLITZ_A_VIDEO_FRAME_RECV_LOG_PATH="${BLITZ_A_VIDEO_FRAME_RECV_LOG_PATH:-$(blitz_dev_component_log_path "a-video-frame-recv")}"
|
||||
export BLITZ_A_VIDEO_DISPLAY_PROBE_LOG_PATH="${BLITZ_A_VIDEO_DISPLAY_PROBE_LOG_PATH:-$(blitz_dev_component_log_path "a-video-display-probe")}"
|
||||
}
|
||||
|
||||
blitz_dev_prepare_bside_logging_env() {
|
||||
blitz_dev_init_instance_context
|
||||
export BLITZ_KCP_STATS_LOG_PATH="${BLITZ_KCP_STATS_LOG_PATH:-$(blitz_dev_component_log_path "b-kcp-session-stats")}"
|
||||
export BLITZ_CONTROL_LATENCY_LOG_PATH="${BLITZ_CONTROL_LATENCY_LOG_PATH:-$(blitz_dev_component_log_path "b-control-latency")}"
|
||||
export BLITZ_VIDEO_STAGE_LOG_PATH="${BLITZ_VIDEO_STAGE_LOG_PATH:-$(blitz_dev_component_log_path "b-video-frame-stages")}"
|
||||
}
|
||||
|
||||
blitz_dev_prepare_5g_logging_env() {
|
||||
blitz_dev_init_instance_context
|
||||
export BLITZ_5G_LINK_LOG_PATH="${BLITZ_5G_LINK_LOG_PATH:-$(blitz_dev_component_log_path "b-5g-link-quality")}"
|
||||
}
|
||||
|
||||
if [[ "${BLITZ_SKIP_DEV_RUN_CONTEXT_INIT:-0}" != "1" && "${BLITZ_BOOT_LOADING_ENV:-0}" != "1" && "${OMNI_BOOT_MODE:-0}" != "1" ]]; then
|
||||
blitz_dev_ensure_run_context
|
||||
elif [[ -f "${BLITZ_RUN_CONTEXT_FILE}" ]]; then
|
||||
blitz_dev_load_run_context || true
|
||||
fi
|
||||
103
robot/v4l2/OmniSocketGo_robot/scripts/dev/log-network-summary.py
Normal file
103
robot/v4l2/OmniSocketGo_robot/scripts/dev/log-network-summary.py
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STOP_REQUESTED = False
|
||||
LOCAL_HTTP_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
|
||||
def handle_signal(signum: int, frame: object) -> None:
|
||||
del signum, frame
|
||||
global STOP_REQUESTED
|
||||
STOP_REQUESTED = True
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Poll /api/network/latest/ and append JSONL snapshots.")
|
||||
parser.add_argument("--url", required=True, help="HTTP endpoint that returns the network summary JSON.")
|
||||
parser.add_argument("--output", required=True, help="Output JSONL path.")
|
||||
parser.add_argument(
|
||||
"--interval-ms",
|
||||
type=int,
|
||||
default=2000,
|
||||
help="Polling interval in milliseconds. Default: 2000.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-timeout-sec",
|
||||
type=float,
|
||||
default=3.0,
|
||||
help="Single request timeout in seconds. Default: 3.0.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def sleep_with_stop(seconds: float) -> None:
|
||||
deadline = time.monotonic() + max(0.0, seconds)
|
||||
while not STOP_REQUESTED:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0.0:
|
||||
return
|
||||
time.sleep(min(remaining, 0.2))
|
||||
|
||||
|
||||
def fetch_json(url: str, timeout_sec: float) -> str:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Cache-Control": "no-cache",
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
# This logger always polls the local backend. Ignore HTTP_PROXY/HTTPS_PROXY
|
||||
# so a developer proxy cannot turn a 127.0.0.1 request into a 502.
|
||||
with LOCAL_HTTP_OPENER.open(request, timeout=timeout_sec) as response:
|
||||
charset = response.headers.get_content_charset("utf-8")
|
||||
payload = response.read().decode(charset)
|
||||
parsed = json.loads(payload)
|
||||
return json.dumps(parsed, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
interval_sec = max(args.interval_ms, 200) / 1000.0
|
||||
output_path = Path(args.output)
|
||||
last_error_log_monotonic = 0.0
|
||||
|
||||
signal.signal(signal.SIGINT, handle_signal)
|
||||
signal.signal(signal.SIGTERM, handle_signal)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with output_path.open("a", encoding="utf-8") as output_file:
|
||||
while not STOP_REQUESTED:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
line = fetch_json(args.url, args.request_timeout_sec)
|
||||
except (TimeoutError, urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError) as error:
|
||||
now = time.monotonic()
|
||||
if now - last_error_log_monotonic >= 10.0:
|
||||
print(f"[network-summary] poll failed: {error}", file=sys.stderr)
|
||||
last_error_log_monotonic = now
|
||||
else:
|
||||
output_file.write(line)
|
||||
output_file.write("\n")
|
||||
output_file.flush()
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
sleep_with_stop(max(0.0, interval_sec - elapsed))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
|
||||
camera_device="${OMNI_CAMERA_DEVICE}"
|
||||
occupancy_policy="${OMNI_CAMERA_OCCUPANCY_POLICY:-check}"
|
||||
release_service="${OMNI_CAMERA_RELEASE_SERVICE:-}"
|
||||
|
||||
die() {
|
||||
echo "[camera-preflight] $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
camera_pids() {
|
||||
fuser "${camera_device}" 2>/dev/null \
|
||||
| tr ' ' '\n' \
|
||||
| grep -E '^[0-9]+$' \
|
||||
| sort -nu \
|
||||
|| true
|
||||
}
|
||||
|
||||
pid_service() {
|
||||
local pid="$1"
|
||||
local service
|
||||
|
||||
service="$(sed -nE 's#.*[/:]([^/:]+\.service)$#\1#p' "/proc/${pid}/cgroup" 2>/dev/null | head -1)"
|
||||
printf '%s' "${service:-unknown}"
|
||||
}
|
||||
|
||||
report_owners() {
|
||||
local pid
|
||||
local comm
|
||||
local service
|
||||
local found=0
|
||||
|
||||
while read -r pid; do
|
||||
[[ -n "${pid}" ]] || continue
|
||||
found=1
|
||||
comm="$(cat "/proc/${pid}/comm" 2>/dev/null || printf 'unknown')"
|
||||
service="$(pid_service "${pid}")"
|
||||
echo "[camera-preflight] owner pid=${pid} command=${comm} service=${service}" >&2
|
||||
done < <(camera_pids)
|
||||
|
||||
if [[ "${found}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
has_proc_manager_owner() {
|
||||
local pid
|
||||
|
||||
while read -r pid; do
|
||||
[[ -n "${pid}" ]] || continue
|
||||
if [[ "$(pid_service "${pid}")" == "proc_manager.service" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done < <(camera_pids)
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ ! -e "${camera_device}" ]]; then
|
||||
die "camera device does not exist: ${camera_device}"
|
||||
fi
|
||||
|
||||
if ! command -v fuser >/dev/null 2>&1; then
|
||||
die "missing required command: fuser (install the psmisc package)"
|
||||
fi
|
||||
|
||||
resolved_device="$(readlink -f "${camera_device}" 2>/dev/null || printf '%s' "${camera_device}")"
|
||||
echo "[camera-preflight] checking ${camera_device} (${resolved_device}) policy=${occupancy_policy}" >&2
|
||||
|
||||
if ! report_owners; then
|
||||
echo "[camera-preflight] ${camera_device} is free" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "${occupancy_policy}" in
|
||||
check)
|
||||
die "${camera_device} is busy; no process was stopped"
|
||||
;;
|
||||
release-known)
|
||||
if [[ -z "${release_service}" ]]; then
|
||||
die "OMNI_CAMERA_RELEASE_SERVICE is required when policy=release-known"
|
||||
fi
|
||||
|
||||
echo "[camera-preflight] stopping known camera service ${release_service}" >&2
|
||||
systemctl stop "${release_service}"
|
||||
|
||||
if ! report_owners; then
|
||||
echo "[camera-preflight] ${camera_device} was released by ${release_service}" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if has_proc_manager_owner; then
|
||||
die "${camera_device} is still owned by proc_manager.service; refusing to stop the whole process manager. Unload the owning ROS component explicitly."
|
||||
fi
|
||||
|
||||
die "${camera_device} remains busy after stopping ${release_service}"
|
||||
;;
|
||||
*)
|
||||
die "unsupported OMNI_CAMERA_OCCUPANCY_POLICY=${occupancy_policy}; expected check or release-known"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export BLITZ_SKIP_DEV_RUN_CONTEXT_INIT="1"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
|
||||
blitz_dev_reset_run_context
|
||||
printf 'run_id=%s\nrun_dir=%s\n' "${BLITZ_RUN_ID}" "${BLITZ_RUN_DIR}"
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
camera_serial="${1:-}"
|
||||
camera_label="${2:-camera}"
|
||||
capture_width="${OMNI_CAMERA_DISCOVERY_WIDTH:-1280}"
|
||||
capture_height="${OMNI_CAMERA_DISCOVERY_HEIGHT:-720}"
|
||||
timeout_sec="${OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC:-10}"
|
||||
|
||||
die() {
|
||||
echo "[camera-discovery] ${camera_label}: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ -n "${camera_serial}" ]] || die "camera serial is required"
|
||||
[[ "${timeout_sec}" =~ ^[0-9]+$ ]] || die "OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC must be a non-negative integer"
|
||||
|
||||
command -v udevadm >/dev/null 2>&1 || die "missing required command: udevadm"
|
||||
command -v v4l2-ctl >/dev/null 2>&1 || die "missing required command: v4l2-ctl"
|
||||
|
||||
shopt -s nullglob
|
||||
deadline=$((SECONDS + timeout_sec))
|
||||
|
||||
while true; do
|
||||
matches=()
|
||||
serial_nodes=()
|
||||
|
||||
for device in /dev/video*; do
|
||||
[[ -c "${device}" ]] || continue
|
||||
device_serial="$(
|
||||
udevadm info --query=property --name="${device}" 2>/dev/null \
|
||||
| sed -n 's/^ID_SERIAL_SHORT=//p' \
|
||||
| head -1
|
||||
)"
|
||||
[[ "${device_serial}" == "${camera_serial}" ]] || continue
|
||||
serial_nodes+=("${device}")
|
||||
|
||||
formats="$(v4l2-ctl -d "${device}" --list-formats-ext 2>/dev/null || true)"
|
||||
grep -q "'MJPG'" <<<"${formats}" || continue
|
||||
grep -q "Size: Discrete ${capture_width}x${capture_height}" <<<"${formats}" || continue
|
||||
matches+=("${device}")
|
||||
done
|
||||
|
||||
if (( ${#matches[@]} == 1 )); then
|
||||
echo "[camera-discovery] ${camera_label}: serial=${camera_serial} -> ${matches[0]} (MJPG ${capture_width}x${capture_height})" >&2
|
||||
printf '%s\n' "${matches[0]}"
|
||||
exit 0
|
||||
fi
|
||||
if (( ${#matches[@]} > 1 )); then
|
||||
die "serial=${camera_serial} matched multiple MJPG nodes: ${matches[*]}"
|
||||
fi
|
||||
if (( SECONDS >= deadline )); then
|
||||
if (( ${#serial_nodes[@]} == 0 )); then
|
||||
die "serial=${camera_serial} was not found under /dev/video*"
|
||||
fi
|
||||
die "serial=${camera_serial} has no MJPG ${capture_width}x${capture_height} node; serial nodes: ${serial_nodes[*]}"
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
87
robot/v4l2/OmniSocketGo_robot/scripts/dev/robot-remote.env
Normal file
87
robot/v4l2/OmniSocketGo_robot/scripts/dev/robot-remote.env
Normal file
@@ -0,0 +1,87 @@
|
||||
# Optional absolute path override for the companion repo.
|
||||
# By default the scripts assume:
|
||||
# OmniSocketGo -> current repo
|
||||
# robot-command-center -> sibling directory next to OmniSocketGo
|
||||
# Example:
|
||||
# ROBOT_COMMAND_CENTER_ROOT="$HOME/Documents/robot-command-center"
|
||||
|
||||
CONTROL_SIDE_OMNISOCKET_SERVER_ADDR="192.168.41.144:10909" # Local LAN hub
|
||||
CONTROL_SIDE_OMNISOCKET_RELAY_VIA="" # No relay
|
||||
|
||||
ROBOT_SIDE_OMNISOCKET_SERVER_ADDR="192.168.41.144:10909" # Local LAN hub
|
||||
ROBOT_SIDE_OMNISOCKET_RELAY_VIA="" # Direct LAN
|
||||
# Log one normal relay packet out of every N packets. Drop events still log immediately.
|
||||
OMNI_RELAY_PACKET_LOG_SAMPLE_EVERY="200"
|
||||
|
||||
CONTROL_WS_ALLOWED_ORIGINS="http://127.0.0.1:5173,http://localhost:5173"
|
||||
VITE_API_BASE_URL="http://127.0.0.1:8001"
|
||||
|
||||
PYTHON3_BIN="python3"
|
||||
PYTHON_VENV_PATH="${OMNISOCKETGO_ROOT}/.venv"
|
||||
|
||||
BACKEND_HOST="0.0.0.0"
|
||||
BACKEND_PORT="8001"
|
||||
OMNISOCKET_TELEMETRY_PEER_ID="peer-a-telemetry"
|
||||
OMNISOCKET_TELEMETRY_INTERVAL_MS="1000"
|
||||
OMNISOCKET_TELEMETRY_STALE_AFTER_MS="3000"
|
||||
OMNI_NETWORK_SUMMARY_LOG_ENABLED="1"
|
||||
OMNI_NETWORK_SUMMARY_LOG_PATH="${OMNISOCKETGO_ROOT}/logs/a-network-summary.jsonl"
|
||||
OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS="1000"
|
||||
OMNI_NETWORK_SUMMARY_LOG_REQUEST_TIMEOUT_SEC="3"
|
||||
|
||||
FRONTEND_HOST="0.0.0.0"
|
||||
FRONTEND_PORT="5173"
|
||||
|
||||
ROS_DISTRO="jazzy"
|
||||
ROBOT_RECEIVER_TRANSPORT="unix_dgram"
|
||||
ROBOT_RECEIVER_SERVER_ADDR="${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR}"
|
||||
ROBOT_RECEIVER_RELAY_VIA="${ROBOT_SIDE_OMNISOCKET_RELAY_VIA}"
|
||||
ROBOT_RECEIVER_PEER_ID="ros-bridge-ctrl"
|
||||
ROBOT_RECEIVER_EXPECTED_SENDER=""
|
||||
ROBOT_RECEIVER_LOCAL_SOCKET_PATH="/tmp/omnisocket-b-side-cmd.sock"
|
||||
ROBOT_RECEIVER_OUTPUT_TOPIC="/hric/robot/cmd_vel"
|
||||
ROBOT_RECEIVER_FRAME_ID="pelvis"
|
||||
ROBOT_RECEIVER_WATCHDOG_TIMEOUT="0.5"
|
||||
ROBOT_RECEIVER_PUBLISH_RATE_HZ="100.0"
|
||||
|
||||
OMNI_VIDEO_PEER_ID="peer-b-video"
|
||||
OMNI_VIDEO_TARGET_PEER="peer-a-video"
|
||||
OMNI_GPSD_HOST="127.0.0.1"
|
||||
OMNI_CAMERA_HEAD_DEVICE="/dev/video26"
|
||||
OMNI_CAMERA_WAIST_DEVICE="/dev/video18"
|
||||
OMNI_CAMERA_AUTO_DISCOVER="1"
|
||||
OMNI_CAMERA_HEAD_SERIAL="CP9E163000H3"
|
||||
OMNI_CAMERA_WAIST_SERIAL="CPCK8530005N"
|
||||
OMNI_CAMERA_DISCOVERY_WIDTH="1280"
|
||||
OMNI_CAMERA_DISCOVERY_HEIGHT="720"
|
||||
OMNI_CAMERA_DISCOVERY_TIMEOUT_SEC="10"
|
||||
OMNI_CAMERA_HEAD_RELEASE_SERVICE="orbbec_head.service"
|
||||
OMNI_CAMERA_WAIST_RELEASE_SERVICE="orbbec_waist.service"
|
||||
OMNI_CAMERA_ACTIVE="head"
|
||||
OMNI_CAMERA_OCCUPANCY_POLICY="release-known"
|
||||
OMNI_CAMERA_PROFILE="day"
|
||||
OMNI_CAMERA_BRIGHTNESS=""
|
||||
OMNI_CAMERA_CUSTOM_CTRL=""
|
||||
OMNI_CAMERA_VERIFY="0"
|
||||
OMNI_VIDEO_SERVER_ADDR="${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR}"
|
||||
OMNI_VIDEO_RELAY_VIA="${ROBOT_SIDE_OMNISOCKET_RELAY_VIA}"
|
||||
OMNI_VIDEO_SOFT_BACKPRESSURE_SEGMENTS="256"
|
||||
OMNI_VIDEO_HARD_BACKPRESSURE_SEGMENTS="1024"
|
||||
OMNI_VIDEO_HARD_BACKPRESSURE_HOLD_MS="5000"
|
||||
OMNI_VIDEO_FRAME_STALL_RECONNECT_MS="30000"
|
||||
OMNI_CONTROL_PEER_ID="peer-b-ctrl"
|
||||
OMNI_CONTROL_EXPECTED_SENDER="peer-a-ctrl"
|
||||
OMNI_CONTROL_SERVER_ADDR="${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR}"
|
||||
OMNI_CONTROL_RELAY_VIA="${ROBOT_SIDE_OMNISOCKET_RELAY_VIA}"
|
||||
OMNI_CONTROL_UNIX_SOCKET_PATH="${ROBOT_RECEIVER_LOCAL_SOCKET_PATH}"
|
||||
OMNI_CONTROL_ACK_PEER_ID="peer-b-ctrl-ack"
|
||||
OMNI_CONTROL_ACK_TARGET_PEER="peer-a-ctrl-ack"
|
||||
BLITZ_CONTROL_ACK_SAMPLE_MOD="10"
|
||||
BLITZ_VIDEO_STAGE_LOG_ENABLED="1"
|
||||
BLITZ_VIDEO_STAGE_LOG_SAMPLE_MOD="10"
|
||||
OMNI_CONTROL_SERVER_IDLE_RECONNECT_MS="30000"
|
||||
|
||||
# A-side backend video freshness guard. Used by scripts/dev/start-backend.sh.
|
||||
OMNI_VIDEO_MAX_FRAME_AGE_MS="1000"
|
||||
|
||||
B_SIDE_OMNID_USE_SUDO="1"
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
|
||||
blitz_dev_prepare_5g_logging_env
|
||||
exec bash "${OMNISOCKETGO_ROOT}/scripts/boot/blitz-5g-link-logger.sh"
|
||||
100
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-b-side-omnid.sh
Normal file
100
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-b-side-omnid.sh
Normal file
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
blitz_dev_prepare_bside_logging_env
|
||||
|
||||
cd "${OMNISOCKETGO_ROOT}"
|
||||
|
||||
export OMNISOCKET_SERVER_ADDR="${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR}"
|
||||
export OMNISOCKET_RELAY_VIA="${ROBOT_SIDE_OMNISOCKET_RELAY_VIA}"
|
||||
export OMNI_VIDEO_SERVER_ADDR="${OMNI_VIDEO_SERVER_ADDR}"
|
||||
export OMNI_VIDEO_RELAY_VIA="${OMNI_VIDEO_RELAY_VIA}"
|
||||
export OMNI_CONTROL_SERVER_ADDR="${OMNI_CONTROL_SERVER_ADDR}"
|
||||
export OMNI_CONTROL_RELAY_VIA="${OMNI_CONTROL_RELAY_VIA}"
|
||||
|
||||
logger_pid=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${logger_pid}" ]]; then
|
||||
kill "${logger_pid}" 2>/dev/null || true
|
||||
wait "${logger_pid}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
start_5g_link_logger_if_needed() {
|
||||
if [[ "${OMNI_5G_LINK_LOG_ENABLED:-1}" != "1" ]]; then
|
||||
echo "[start-b-side-omnid] 5G link logger disabled" >&2
|
||||
return 0
|
||||
fi
|
||||
if [[ "${OMNI_BOOT_MODE:-0}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
bash "${SCRIPT_DIR}/start-5g-link-logger.sh" &
|
||||
logger_pid=$!
|
||||
echo "[start-b-side-omnid] 5G link logger -> ${BLITZ_5G_LINK_LOG_PATH:-unset}" >&2
|
||||
}
|
||||
|
||||
release_known_camera_service() {
|
||||
local service="$1"
|
||||
|
||||
if [[ "${OMNI_CAMERA_OCCUPANCY_POLICY:-check}" != "release-known" || -z "${service}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if systemctl is-active --quiet "${service}"; then
|
||||
echo "[start-b-side-omnid] stopping known camera service ${service} before discovery" >&2
|
||||
systemctl stop "${service}"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_camera_devices() {
|
||||
if [[ "${OMNI_CAMERA_AUTO_DISCOVER:-0}" != "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
OMNI_CAMERA_HEAD_DEVICE="$(
|
||||
bash "${SCRIPT_DIR}/resolve-camera-device.sh" "${OMNI_CAMERA_HEAD_SERIAL}" head
|
||||
)"
|
||||
OMNI_CAMERA_WAIST_DEVICE="$(
|
||||
bash "${SCRIPT_DIR}/resolve-camera-device.sh" "${OMNI_CAMERA_WAIST_SERIAL}" waist
|
||||
)"
|
||||
if [[ "${OMNI_CAMERA_HEAD_DEVICE}" == "${OMNI_CAMERA_WAIST_DEVICE}" ]]; then
|
||||
echo "[start-b-side-omnid] head and waist resolved to the same device: ${OMNI_CAMERA_HEAD_DEVICE}" >&2
|
||||
return 1
|
||||
fi
|
||||
export OMNI_CAMERA_HEAD_DEVICE OMNI_CAMERA_WAIST_DEVICE
|
||||
echo "[start-b-side-omnid] resolved head=${OMNI_CAMERA_HEAD_DEVICE} waist=${OMNI_CAMERA_WAIST_DEVICE}" >&2
|
||||
}
|
||||
|
||||
if [[ ! -x "./bin/b_side_omnid" ]]; then
|
||||
if [[ "${OMNI_BOOT_MODE:-0}" == "1" ]]; then
|
||||
echo "Missing ./bin/b_side_omnid in boot mode; build it before enabling the autostart service." >&2
|
||||
exit 1
|
||||
fi
|
||||
make b_side_omnid
|
||||
fi
|
||||
|
||||
launch_b_side_omnid() {
|
||||
trap cleanup EXIT INT TERM
|
||||
start_5g_link_logger_if_needed
|
||||
release_known_camera_service "${OMNI_CAMERA_HEAD_RELEASE_SERVICE:-orbbec_head.service}"
|
||||
release_known_camera_service "${OMNI_CAMERA_WAIST_RELEASE_SERVICE:-orbbec_waist.service}"
|
||||
resolve_camera_devices
|
||||
OMNI_CAMERA_DEVICE="${OMNI_CAMERA_HEAD_DEVICE}" \
|
||||
OMNI_CAMERA_RELEASE_SERVICE="${OMNI_CAMERA_HEAD_RELEASE_SERVICE:-orbbec_head.service}" \
|
||||
bash "${SCRIPT_DIR}/prepare-camera-device.sh"
|
||||
OMNI_CAMERA_DEVICE="${OMNI_CAMERA_WAIST_DEVICE}" \
|
||||
OMNI_CAMERA_RELEASE_SERVICE="${OMNI_CAMERA_WAIST_RELEASE_SERVICE:-orbbec_waist.service}" \
|
||||
bash "${SCRIPT_DIR}/prepare-camera-device.sh"
|
||||
OMNI_CAMERA_DEVICE="${OMNI_CAMERA_HEAD_DEVICE}" bash "${SCRIPT_DIR}/apply-camera-controls.sh"
|
||||
OMNI_CAMERA_DEVICE="${OMNI_CAMERA_WAIST_DEVICE}" bash "${SCRIPT_DIR}/apply-camera-controls.sh"
|
||||
./bin/b_side_omnid
|
||||
}
|
||||
|
||||
if [[ "${B_SIDE_OMNID_USE_SUDO}" == "1" && "${EUID}" -ne 0 ]]; then
|
||||
exec sudo -E bash -lc 'cd "$1" && export B_SIDE_OMNID_USE_SUDO=0 && exec bash "$2"' _ "${OMNISOCKETGO_ROOT}" "${SCRIPT_DIR}/start-b-side-omnid.sh"
|
||||
fi
|
||||
|
||||
launch_b_side_omnid
|
||||
60
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-backend.sh
Normal file
60
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-backend.sh
Normal file
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
require_robot_command_center_root
|
||||
blitz_dev_prepare_backend_logging_env
|
||||
|
||||
if [[ ! -x "${PYTHON_VENV_PATH}/bin/python" ]]; then
|
||||
echo "[start-backend] creating or repairing virtualenv at ${PYTHON_VENV_PATH}" >&2
|
||||
"${PYTHON3_BIN}" -m venv "${PYTHON_VENV_PATH}"
|
||||
fi
|
||||
|
||||
if [[ ! -x "${PYTHON_VENV_PATH}/bin/python" ]]; then
|
||||
echo "[start-backend] virtualenv is incomplete: missing ${PYTHON_VENV_PATH}/bin/python" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "${PYTHON_VENV_PATH}/bin/activate"
|
||||
|
||||
cd "${BACKEND_DIR}"
|
||||
export OMNISOCKET_SERVER_ADDR="${CONTROL_SIDE_OMNISOCKET_SERVER_ADDR}"
|
||||
export OMNISOCKET_RELAY_VIA="${CONTROL_SIDE_OMNISOCKET_RELAY_VIA}"
|
||||
|
||||
logger_pid=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${logger_pid}" ]]; then
|
||||
kill "${logger_pid}" 2>/dev/null || true
|
||||
wait "${logger_pid}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
start_network_summary_logger() {
|
||||
local logger_url
|
||||
local logger_dir
|
||||
|
||||
if [[ "${OMNI_NETWORK_SUMMARY_LOG_ENABLED}" != "1" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
logger_url="http://127.0.0.1:${BACKEND_PORT}/api/network/latest/"
|
||||
logger_dir="$(dirname "${OMNI_NETWORK_SUMMARY_LOG_PATH}")"
|
||||
mkdir -p "${logger_dir}"
|
||||
|
||||
python "${SCRIPT_DIR}/log-network-summary.py" \
|
||||
--url "${logger_url}" \
|
||||
--output "${OMNI_NETWORK_SUMMARY_LOG_PATH}" \
|
||||
--interval-ms "${OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS}" \
|
||||
--request-timeout-sec "${OMNI_NETWORK_SUMMARY_LOG_REQUEST_TIMEOUT_SEC}" &
|
||||
logger_pid=$!
|
||||
echo "[start-backend] network summary logger -> ${OMNI_NETWORK_SUMMARY_LOG_PATH} (${OMNI_NETWORK_SUMMARY_LOG_INTERVAL_MS} ms)" >&2
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
start_network_summary_logger
|
||||
python -m uvicorn config.asgi:application --host "${BACKEND_HOST}" --port "${BACKEND_PORT}"
|
||||
21
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-dev-tmux.sh
Normal file
21
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-dev-tmux.sh
Normal file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SESSION_NAME="${1:-robot-remote}"
|
||||
|
||||
if ! command -v tmux >/dev/null 2>&1; then
|
||||
echo "tmux is required for this launcher" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if tmux has-session -t "${SESSION_NAME}" 2>/dev/null; then
|
||||
exec tmux attach -t "${SESSION_NAME}"
|
||||
fi
|
||||
|
||||
tmux new-session -d -s "${SESSION_NAME}" -n backend "bash -lc '${SCRIPT_DIR}/start-backend.sh'"
|
||||
tmux new-window -t "${SESSION_NAME}:" -n frontend "bash -lc '${SCRIPT_DIR}/start-frontend.sh'"
|
||||
tmux new-window -t "${SESSION_NAME}:" -n ros "bash -lc '${SCRIPT_DIR}/start-ros-receiver.sh'"
|
||||
tmux new-window -t "${SESSION_NAME}:" -n b-side "bash -lc '${SCRIPT_DIR}/start-b-side-omnid.sh'"
|
||||
|
||||
exec tmux attach -t "${SESSION_NAME}"
|
||||
10
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-frontend.sh
Normal file
10
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-frontend.sh
Normal file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
require_robot_command_center_root
|
||||
|
||||
cd "${FRONTEND_DIR}"
|
||||
exec npm run dev -- --host "${FRONTEND_HOST}" --port "${FRONTEND_PORT}"
|
||||
22
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-local-hub.sh
Normal file
22
robot/v4l2/OmniSocketGo_robot/scripts/dev/start-local-hub.sh
Normal file
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
|
||||
hub_binary="${OMNISOCKETGO_ROOT}/bin/kcpserver"
|
||||
hub_listen_addr="${LOCAL_HUB_LISTEN_ADDR:-0.0.0.0:10909}"
|
||||
telemetry_peer_id="${LOCAL_HUB_TELEMETRY_PEER_ID:-peer-a-telemetry}"
|
||||
telemetry_interval="${LOCAL_HUB_TELEMETRY_INTERVAL:-1000ms}"
|
||||
|
||||
if [[ ! -x "${hub_binary}" ]]; then
|
||||
echo "[start-local-hub] missing executable ${hub_binary}; run: make bin/kcpserver" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[start-local-hub] listen=${hub_listen_addr} relay=disabled telemetry_peer=${telemetry_peer_id}" >&2
|
||||
exec "${hub_binary}" \
|
||||
-listen "${hub_listen_addr}" \
|
||||
-telemetry-peer "${telemetry_peer_id}" \
|
||||
-telemetry-interval "${telemetry_interval}"
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source_with_nounset_off() {
|
||||
set +u
|
||||
# shellcheck disable=SC1090
|
||||
source "$1"
|
||||
set -u
|
||||
}
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/load-env.sh"
|
||||
if [[ ! -f "/opt/ros/${ROS_DISTRO}/setup.bash" ]]; then
|
||||
echo "Missing ROS distro setup: /opt/ros/${ROS_DISTRO}/setup.bash" >&2
|
||||
exit 1
|
||||
fi
|
||||
source_with_nounset_off "/opt/ros/${ROS_DISTRO}/setup.bash"
|
||||
|
||||
cd "${ROS_CONTROL_PY_DIR}"
|
||||
if [[ ! -f "install/setup.bash" ]]; then
|
||||
echo "Missing ROS workspace setup: ${ROS_CONTROL_PY_DIR}/install/setup.bash" >&2
|
||||
exit 1
|
||||
fi
|
||||
source_with_nounset_off "install/setup.bash"
|
||||
|
||||
launch_args=(
|
||||
"transport:=${ROBOT_RECEIVER_TRANSPORT}"
|
||||
"peer_id:=${ROBOT_RECEIVER_PEER_ID}"
|
||||
"local_socket_path:=${ROBOT_RECEIVER_LOCAL_SOCKET_PATH}"
|
||||
"output_topic:=${ROBOT_RECEIVER_OUTPUT_TOPIC}"
|
||||
"frame_id:=${ROBOT_RECEIVER_FRAME_ID}"
|
||||
"watchdog_timeout:=${ROBOT_RECEIVER_WATCHDOG_TIMEOUT}"
|
||||
"publish_rate_hz:=${ROBOT_RECEIVER_PUBLISH_RATE_HZ}"
|
||||
)
|
||||
|
||||
if [[ -n "${ROBOT_RECEIVER_SERVER_ADDR}" ]]; then
|
||||
launch_args+=("server_addr:=${ROBOT_RECEIVER_SERVER_ADDR}")
|
||||
fi
|
||||
|
||||
if [[ -n "${ROBOT_RECEIVER_RELAY_VIA}" ]]; then
|
||||
launch_args+=("relay_via:=${ROBOT_RECEIVER_RELAY_VIA}")
|
||||
fi
|
||||
|
||||
if [[ -n "${ROBOT_RECEIVER_EXPECTED_SENDER}" ]]; then
|
||||
launch_args+=("expected_sender:=${ROBOT_RECEIVER_EXPECTED_SENDER}")
|
||||
fi
|
||||
|
||||
exec ros2 launch udp_teleop_bridge robot_udp_receiver.launch.py "${launch_args[@]}"
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Send high-rate control packets to benchmark the KCP control session."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import yaml
|
||||
|
||||
from omnisocket_control import make_control_packet
|
||||
|
||||
try:
|
||||
from omnisocket import CONTROL_DEFAULTS, Session
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(ROOT / "python"))
|
||||
from omnisocket import CONTROL_DEFAULTS, Session
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
config_path = ROOT / "config" / "omnisocket_demo.yaml"
|
||||
if not config_path.exists():
|
||||
return {}
|
||||
with config_path.open("r", encoding="utf-8") as file:
|
||||
return yaml.safe_load(file) or {}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--rate", type=float, default=200.0, help="send rate in Hz")
|
||||
parser.add_argument("--count", type=int, default=1000, help="packets to send")
|
||||
args = parser.parse_args()
|
||||
|
||||
config = load_config()
|
||||
transport_cfg = config.get("transport", {})
|
||||
sender_cfg = config.get("control_sender", {})
|
||||
|
||||
session = Session()
|
||||
session.connect(
|
||||
server_addr=str(transport_cfg.get("server_addr", "127.0.0.1:10909")),
|
||||
peer_id=str(sender_cfg.get("peer_id", "peer-a-ctrl")),
|
||||
relay_via=str(transport_cfg.get("relay_via", "")),
|
||||
bind_ip=str(transport_cfg.get("bind_ip", "")),
|
||||
bind_device=str(transport_cfg.get("bind_device", "")),
|
||||
**CONTROL_DEFAULTS,
|
||||
)
|
||||
|
||||
target_peer = str(sender_cfg.get("target_peer", "peer-b-ctrl"))
|
||||
spacing = 1.0 / args.rate if args.rate > 0 else 0.0
|
||||
start = time.perf_counter()
|
||||
|
||||
try:
|
||||
for seq_id in range(args.count):
|
||||
packet = make_control_packet(seq_id, "set_surge", drive_value=0.25)
|
||||
session.send(to=target_peer, data=packet.encode())
|
||||
if spacing > 0:
|
||||
target = start + (seq_id + 1) * spacing
|
||||
remaining = target - time.perf_counter()
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
finally:
|
||||
elapsed = time.perf_counter() - start
|
||||
print(
|
||||
f"sent {args.count} control packets in {elapsed:.3f}s "
|
||||
f"({(args.count / elapsed) if elapsed > 0 else 0.0:.1f} pkt/s)"
|
||||
)
|
||||
print(f"stats={session.stats()}")
|
||||
session.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
136
robot/v4l2/OmniSocketGo_robot/scripts/refresh-latency-summary.sh
Normal file
136
robot/v4l2/OmniSocketGo_robot/scripts/refresh-latency-summary.sh
Normal file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
remote_source="boll@175.178.116.187:/home/boll/LMJWork/OmniSocketGo/peer-b-latency.jsonl"
|
||||
local_peer_a="$repo_dir/peer-a-latency.jsonl"
|
||||
local_peer_b="$repo_dir/peer-b-latency.jsonl"
|
||||
summary_output="$repo_dir/latency-summary.jsonl"
|
||||
chart_output="$repo_dir/latency-summary.html"
|
||||
latency_binary="$repo_dir/bin/latencysummary"
|
||||
go_cache_dir="${GOCACHE:-/tmp/omnisocketgo-go-build}"
|
||||
poll_interval_seconds=1
|
||||
|
||||
remote_tmp=""
|
||||
summary_tmp=""
|
||||
chart_tmp=""
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
|
||||
}
|
||||
|
||||
cleanup_temp_file() {
|
||||
local path="$1"
|
||||
if [[ -n "$path" && -e "$path" ]]; then
|
||||
rm -f "$path"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
cleanup_temp_file "$remote_tmp"
|
||||
cleanup_temp_file "$summary_tmp"
|
||||
cleanup_temp_file "$chart_tmp"
|
||||
}
|
||||
|
||||
handle_interrupt() {
|
||||
log "received interrupt signal, stopping refresh loop"
|
||||
cleanup
|
||||
exit 130
|
||||
}
|
||||
|
||||
handle_terminate() {
|
||||
log "received terminate signal, stopping refresh loop"
|
||||
cleanup
|
||||
exit 143
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap handle_interrupt INT
|
||||
trap handle_terminate TERM
|
||||
|
||||
cd "$repo_dir" || exit 1
|
||||
|
||||
mkdir -p "$repo_dir/bin"
|
||||
mkdir -p "$go_cache_dir"
|
||||
if ! GOCACHE="$go_cache_dir" go build -o "$latency_binary" ./cmd/latencysummary; then
|
||||
log "build failed; exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "starting 1-second refresh loop"
|
||||
|
||||
while true; do
|
||||
remote_tmp="$(mktemp "$repo_dir/peer-b-latency.jsonl.tmp.XXXXXX")" || exit 1
|
||||
if scp -P 10022 "$remote_source" "$remote_tmp"; then
|
||||
if mv -f "$remote_tmp" "$local_peer_b"; then
|
||||
remote_tmp=""
|
||||
else
|
||||
status=$?
|
||||
log "failed to replace $(basename "$local_peer_b") after scp (exit $status)"
|
||||
cleanup_temp_file "$remote_tmp"
|
||||
remote_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
status=$?
|
||||
log "scp refresh failed (exit $status)"
|
||||
cleanup_temp_file "$remote_tmp"
|
||||
remote_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
|
||||
summary_tmp="$(mktemp "$repo_dir/latency-summary.tmp.XXXXXX.jsonl")" || exit 1
|
||||
chart_tmp="${summary_tmp%.jsonl}.html"
|
||||
if "$latency_binary" \
|
||||
-input "$local_peer_a" \
|
||||
-input "$local_peer_b" \
|
||||
-shared-max-offset 1 \
|
||||
-output "$summary_tmp"; then
|
||||
if [[ ! -f "$summary_tmp" || ! -f "$chart_tmp" ]]; then
|
||||
log "summary succeeded but temporary outputs are incomplete"
|
||||
cleanup_temp_file "$summary_tmp"
|
||||
cleanup_temp_file "$chart_tmp"
|
||||
summary_tmp=""
|
||||
chart_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! mv -f "$summary_tmp" "$summary_output"; then
|
||||
status=$?
|
||||
log "failed to replace $(basename "$summary_output") (exit $status)"
|
||||
cleanup_temp_file "$summary_tmp"
|
||||
cleanup_temp_file "$chart_tmp"
|
||||
summary_tmp=""
|
||||
chart_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
summary_tmp=""
|
||||
|
||||
if ! mv -f "$chart_tmp" "$chart_output"; then
|
||||
status=$?
|
||||
log "failed to replace $(basename "$chart_output") (exit $status)"
|
||||
cleanup_temp_file "$chart_tmp"
|
||||
chart_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
chart_tmp=""
|
||||
else
|
||||
status=$?
|
||||
log "latency summary refresh failed (exit $status)"
|
||||
cleanup_temp_file "$summary_tmp"
|
||||
cleanup_temp_file "$chart_tmp"
|
||||
summary_tmp=""
|
||||
chart_tmp=""
|
||||
sleep "$poll_interval_seconds"
|
||||
continue
|
||||
fi
|
||||
|
||||
sleep "$poll_interval_seconds"
|
||||
done
|
||||
1202
robot/v4l2/OmniSocketGo_robot/scripts/run-kcp-batch-test.sh
Normal file
1202
robot/v4l2/OmniSocketGo_robot/scripts/run-kcp-batch-test.sh
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user