#!/usr/bin/env bash set -euo pipefail BOOT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEV_SCRIPT_DIR="$(cd "${BOOT_SCRIPT_DIR}/../dev" && pwd)" source_with_nounset_off() { set +u # shellcheck disable=SC1090 source "$1" set -u } blitz_host_from_addr() { local value="${1:-}" if [[ -z "${value}" ]]; then return 1 fi if [[ "${value}" == \[*\]:* ]]; then value="${value#\[}" printf '%s\n' "${value%%]:*}" return 0 fi printf '%s\n' "${value%%:*}" } blitz_load_boot_env() { local env_file local default_time_server if [[ "${BLITZ_BOOT_ENV_LOADED:-0}" == "1" ]]; then return 0 fi export BLITZ_BOOT_LOADING_ENV="1" # shellcheck disable=SC1091 source "${DEV_SCRIPT_DIR}/load-env.sh" unset BLITZ_BOOT_LOADING_ENV for env_file in \ "${BOOT_SCRIPT_DIR}/robot-boot.env" \ "${BOOT_SCRIPT_DIR}/robot-boot.env.local" do if [[ -f "${env_file}" ]]; then set -a # shellcheck disable=SC1090 source "${env_file}" set +a fi done if declare -F normalize_loaded_env_vars >/dev/null 2>&1; then normalize_loaded_env_vars fi default_time_server="$(blitz_host_from_addr "${ROBOT_SIDE_OMNISOCKET_SERVER_ADDR:-}" || true)" export BLITZ_BOOT_DELAY_SEC="${BLITZ_BOOT_DELAY_SEC:-30}" export BLITZ_RUN_ROOT="${BLITZ_RUN_ROOT:-/var/log/blitz-robot}" export BLITZ_LOG_FILE="${BLITZ_LOG_FILE:-/var/log/blitz-robot/startup.log}" export BLITZ_RUNTIME_DIR="${BLITZ_RUNTIME_DIR:-/run/blitz-robot}" export BLITZ_RUN_CONTEXT_FILE="${BLITZ_RUN_CONTEXT_FILE:-${BLITZ_RUNTIME_DIR}/run-context.env}" export BLITZ_RUN_ID_FILE="${BLITZ_RUN_ID_FILE:-${BLITZ_RUNTIME_DIR}/run-id}" export BLITZ_CURRENT_RUN_LINK="${BLITZ_CURRENT_RUN_LINK:-${BLITZ_RUN_ROOT}/current}" export BLITZ_5G_DIAL_DIR="${BLITZ_5G_DIAL_DIR:-${BOOT_SCRIPT_DIR}}" export BLITZ_5G_SERIAL_PORT="${BLITZ_5G_SERIAL_PORT:-/dev/ttyUSB7}" export BLITZ_5G_INTERFACE="${BLITZ_5G_INTERFACE:-}" export BLITZ_5G_MODEM_SUBNET="${BLITZ_5G_MODEM_SUBNET:-192.168.224.0/22}" export BLITZ_5G_GATEWAY="${BLITZ_5G_GATEWAY:-192.168.225.1}" export BLITZ_5G_SKIP_DHCP="${BLITZ_5G_SKIP_DHCP:-0}" export BLITZ_5G_REMOVE_DEFAULT_ROUTE="${BLITZ_5G_REMOVE_DEFAULT_ROUTE:-1}" export BLITZ_5G_ROUTE_TARGETS="${BLITZ_5G_ROUTE_TARGETS:-106.55.173.235}" export BLITZ_5G_INFO_JSON="${BLITZ_5G_INFO_JSON:-${BLITZ_5G_DIAL_DIR}/modem_network_info.json}" export BLITZ_5G_DISABLE_INTERFACES="${BLITZ_5G_DISABLE_INTERFACES:-}" export BLITZ_5G_SERIAL_WAIT_SEC="${BLITZ_5G_SERIAL_WAIT_SEC:-60}" export BLITZ_5G_ROUTE_WAIT_SEC="${BLITZ_5G_ROUTE_WAIT_SEC:-30}" export BLITZ_TIME_SERVER_IP="${BLITZ_TIME_SERVER_IP:-${default_time_server}}" export BLITZ_ROS_USER="${BLITZ_ROS_USER:-nvidia}" export BLITZ_ROS_SOCKET_WAIT_SEC="${BLITZ_ROS_SOCKET_WAIT_SEC:-20}" export BLITZ_WATCHDOG_INTERVAL_SEC="${BLITZ_WATCHDOG_INTERVAL_SEC:-5}" export BLITZ_HEALTH_STALE_SEC="${BLITZ_HEALTH_STALE_SEC:-15}" export BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC="${BLITZ_OMNID_THREAD_HEARTBEAT_TIMEOUT_SEC:-15}" export BLITZ_KCP_STATS_INTERVAL_MS="${BLITZ_KCP_STATS_INTERVAL_MS:-1000}" export BLITZ_CONTROL_LATENCY_LOG_ENABLED="${BLITZ_CONTROL_LATENCY_LOG_ENABLED:-1}" export BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD="${BLITZ_CONTROL_LATENCY_LOG_SAMPLE_MOD:-100}" export BLITZ_5G_LINK_LOG_INTERVAL_SEC="${BLITZ_5G_LINK_LOG_INTERVAL_SEC:-5}" export BLITZ_JSONL_FLUSH_INTERVAL_MS="${BLITZ_JSONL_FLUSH_INTERVAL_MS:-1000}" export BLITZ_JSONL_FLUSH_BYTES="${BLITZ_JSONL_FLUSH_BYTES:-262144}" export BLITZ_JSONL_ROTATE_BYTES="${BLITZ_JSONL_ROTATE_BYTES:-134217728}" export BLITZ_JSONL_ROTATE_FILES="${BLITZ_JSONL_ROTATE_FILES:-8}" export BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC="${BLITZ_INCIDENT_COMMAND_TIMEOUT_SEC:-5}" export BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC="${BLITZ_INCIDENT_TOTAL_TIMEOUT_SEC:-30}" export BLITZ_NETWORK_FAIL_THRESHOLD="${BLITZ_NETWORK_FAIL_THRESHOLD:-3}" export BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC="${BLITZ_NETWORK_RECOVERY_COOLDOWN_SEC:-30}" export BLITZ_GPS_MONITOR_ENABLED="${BLITZ_GPS_MONITOR_ENABLED:-1}" export BLITZ_GPS_DEVICE_GLOB="${BLITZ_GPS_DEVICE_GLOB:-/dev/ttyCH341USB*}" export BLITZ_GPS_CHECK_INTERVAL_SEC="${BLITZ_GPS_CHECK_INTERVAL_SEC:-10}" export BLITZ_GPS_RESTART_UNITS="${BLITZ_GPS_RESTART_UNITS:-gpsd.socket gpsd.service}" export BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION="${BLITZ_WATCHDOG_ALLOW_FAULT_INJECTION:-0}" export BLITZ_BOOT_ENV_LOADED="1" } blitz_timestamp() { date '+%Y-%m-%d %H:%M:%S%z' } blitz_sanitize_detail() { local detail="${1:-}" detail="${detail//$'\n'/ ; }" detail="${detail//$'\r'/ }" printf '%s' "${detail}" } blitz_log() { local step="${1:-unknown-step}" local action="${2:-unknown-action}" local result="${3:-info}" local details="${4:-}" local exit_code="${5:-0}" printf '%s | %s | %s | %s | %s | %s\n' \ "$(blitz_timestamp)" \ "${step}" \ "${action}" \ "${result}" \ "$(blitz_sanitize_detail "${details}")" \ "${exit_code}" } blitz_join_cmd() { local cmd=() local arg for arg in "$@"; do cmd+=("$(printf '%q' "${arg}")") done printf '%s' "${cmd[*]}" } blitz_require_command() { local command_name="$1" local step="${2:-precheck}" if command -v "${command_name}" >/dev/null 2>&1; then blitz_log "${step}" "require-command" "success" "command=${command_name}" 0 return 0 fi blitz_log "${step}" "require-command" "failure" "missing command: ${command_name}" 127 return 127 } blitz_require_file() { local path="$1" local step="${2:-precheck}" if [[ -f "${path}" ]]; then blitz_log "${step}" "require-file" "success" "path=${path}" 0 return 0 fi blitz_log "${step}" "require-file" "failure" "missing file: ${path}" 1 return 1 } blitz_require_executable() { local path="$1" local step="${2:-precheck}" if [[ -x "${path}" ]]; then blitz_log "${step}" "require-executable" "success" "path=${path}" 0 return 0 fi blitz_log "${step}" "require-executable" "failure" "missing executable: ${path}" 1 return 1 } blitz_require_root() { local step="${1:-precheck}" if [[ "${EUID}" -eq 0 ]]; then blitz_log "${step}" "require-root" "success" "uid=${EUID}" 0 return 0 fi blitz_log "${step}" "require-root" "failure" "root privileges are required" 1 return 1 } blitz_run() { local step="$1" local action="$2" local rc shift 2 blitz_log "${step}" "${action}" "start" "$(blitz_join_cmd "$@")" 0 if "$@"; then blitz_log "${step}" "${action}" "success" "$(blitz_join_cmd "$@")" 0 return 0 else rc=$? fi blitz_log "${step}" "${action}" "failure" "$(blitz_join_cmd "$@")" "${rc}" return "${rc}" } blitz_route_ready() { local target_ip="$1" local expected_interface="${2:-}" local route_output route_output="$(ip route get "${target_ip}" 2>&1 || true)" if [[ -z "${route_output}" ]]; then return 1 fi if [[ "${route_output}" == *"unreachable"* || "${route_output}" == *"prohibit"* ]]; then return 1 fi if [[ -n "${expected_interface}" && "${route_output}" != *" dev ${expected_interface} "* && "${route_output}" != *" dev ${expected_interface}" ]]; then return 1 fi printf '%s\n' "${route_output}" return 0 } blitz_interface_exists() { local interface_name="${1:-}" if [[ -z "${interface_name}" ]]; then return 1 fi ip link show dev "${interface_name}" >/dev/null 2>&1 } blitz_read_5g_info_interface() { local info_json="$1" if [[ -z "${info_json}" || ! -f "${info_json}" ]]; then return 1 fi python3 - "${info_json}" <<'PY' import json import sys path = sys.argv[1] try: with open(path, "r", encoding="utf-8") as handle: payload = json.load(handle) except Exception: raise SystemExit(1) interface = str(payload.get("interface") or "").strip() if not interface: raise SystemExit(1) print(interface) PY } blitz_detect_5g_interface_from_subnet() { local modem_subnet="${1:-${BLITZ_5G_MODEM_SUBNET:-}}" if [[ -z "${modem_subnet}" ]]; then return 1 fi python3 - "${modem_subnet}" <<'PY' import ipaddress import json import subprocess import sys subnet = ipaddress.ip_network(sys.argv[1], strict=False) skip = {"lo", "docker0", "l4tbr0"} def priority(name: str) -> tuple[int, str]: if name.startswith("enx"): return (0, name) if name.startswith("wwan"): return (1, name) if name.startswith("usb"): return (2, name) if name.startswith("eth"): return (3, name) return (9, name) try: output = subprocess.check_output(["ip", "-j", "-4", "addr", "show"], text=True) payload = json.loads(output) except Exception: raise SystemExit(1) candidates = [] for item in payload: ifname = str(item.get("ifname") or "").strip() if not ifname or ifname in skip: continue for addr in item.get("addr_info") or []: if addr.get("family") != "inet": continue local = addr.get("local") prefixlen = addr.get("prefixlen") if not local or prefixlen is None: continue try: iface = ipaddress.ip_interface(f"{local}/{prefixlen}") except ValueError: continue if iface.ip in subnet: candidates.append((priority(ifname), ifname)) break if not candidates: raise SystemExit(1) candidates.sort(key=lambda item: item[0]) print(candidates[0][1]) PY } blitz_refresh_5g_info_json() { local interface_name="$1" local info_json="${2:-${BLITZ_5G_INFO_JSON:-}}" if [[ -z "${interface_name}" || -z "${info_json}" ]]; then return 1 fi python3 - "${interface_name}" "${info_json}" <<'PY' import json import os import subprocess import sys interface_name = sys.argv[1] path = sys.argv[2] try: output = subprocess.check_output(["ip", "-j", "addr", "show", "dev", interface_name], text=True) payload = json.loads(output) except Exception: raise SystemExit(1) if not payload: raise SystemExit(1) item = payload[0] ipv4 = [] ipv6 = [] for addr in item.get("addr_info") or []: local = addr.get("local") prefixlen = addr.get("prefixlen") family = addr.get("family") if not local or prefixlen is None: continue entry = f"{local}/{prefixlen}" if family == "inet": ipv4.append(entry) elif family == "inet6": ipv6.append(entry) data = { "interface": interface_name, "ipv4": ipv4, "ipv6": ipv6, } parent = os.path.dirname(path) if parent: os.makedirs(parent, exist_ok=True) temp_path = f"{path}.tmp.{os.getpid()}" with open(temp_path, "w", encoding="utf-8") as handle: json.dump(data, handle, ensure_ascii=False, indent=2) os.replace(temp_path, path) PY } blitz_resolve_5g_interface() { local explicit_interface="${BLITZ_5G_INTERFACE:-}" local info_json="${BLITZ_5G_INFO_JSON:-}" local recorded_interface="" local detected_interface="" if [[ -n "${explicit_interface}" ]]; then if blitz_interface_exists "${explicit_interface}"; then printf '%s\n' "${explicit_interface}" return 0 fi return 1 fi recorded_interface="$(blitz_read_5g_info_interface "${info_json}" || true)" if [[ -n "${recorded_interface}" ]] && blitz_interface_exists "${recorded_interface}"; then printf '%s\n' "${recorded_interface}" return 0 fi detected_interface="$(blitz_detect_5g_interface_from_subnet || true)" if [[ -n "${detected_interface}" ]]; then if [[ "${detected_interface}" != "${recorded_interface}" ]]; then blitz_refresh_5g_info_json "${detected_interface}" "${info_json}" >/dev/null 2>&1 || true fi printf '%s\n' "${detected_interface}" return 0 fi return 1 } blitz_prepare_runtime_dir() { local runtime_dir blitz_load_boot_env runtime_dir="${BLITZ_RUNTIME_DIR}" mkdir -p "${runtime_dir}" if [[ "${EUID}" -eq 0 ]]; then chown "root:${BLITZ_ROS_USER}" "${runtime_dir}" chmod 0775 "${runtime_dir}" else chmod 0775 "${runtime_dir}" 2>/dev/null || true fi blitz_log "runtime-dir" "prepare" "success" "path=${runtime_dir}" 0 } blitz_prepare_run_root() { local run_root local run_dir local incidents_dir blitz_load_boot_env run_root="${BLITZ_RUN_ROOT}" run_dir="${run_root}/runs" incidents_dir="${run_root}/incidents" mkdir -p "${run_dir}" "${incidents_dir}" if [[ "${EUID}" -eq 0 ]]; then chown -R "root:${BLITZ_ROS_USER}" "${run_root}" 2>/dev/null || true chmod 0775 "${run_root}" "${run_dir}" "${incidents_dir}" 2>/dev/null || true fi } blitz_load_run_context_env() { local context_file="${1:-${BLITZ_RUN_CONTEXT_FILE:-}}" if [[ -z "${context_file}" || ! -f "${context_file}" ]]; then return 1 fi set -a # shellcheck disable=SC1090 source "${context_file}" set +a return 0 } blitz_read_run_id() { local run_id_file="${BLITZ_RUN_ID_FILE:-}" if [[ -z "${run_id_file}" || ! -f "${run_id_file}" ]]; then return 1 fi tr -d '\r\n' < "${run_id_file}" } blitz_utc_compact_timestamp() { date -u '+%Y%m%dT%H%M%SZ' } blitz_new_run_id() { printf '%s\n' "$(blitz_utc_compact_timestamp)" } blitz_new_incident_id() { local prefix="${1:-incident}" printf '%s-%s-%d\n' "${prefix}" "$(blitz_utc_compact_timestamp)" "$$" } blitz_new_instance_id() { printf '%s-%d\n' "$(blitz_utc_compact_timestamp)" "$$" } blitz_git_commit() { git -C "${OMNISOCKETGO_ROOT}" rev-parse HEAD 2>/dev/null || true } blitz_git_dirty_flag() { if git -C "${OMNISOCKETGO_ROOT}" diff --quiet --ignore-submodules=dirty >/dev/null 2>&1; then printf '0\n' return 0 fi printf '1\n' } blitz_write_run_context() { local run_id="$1" local run_dir="$2" local boot_id="$3" local context_file="${BLITZ_RUN_CONTEXT_FILE}" local id_file="${BLITZ_RUN_ID_FILE}" local temp_context local temp_info local commit_hash local dirty_flag local started_at commit_hash="$(blitz_git_commit)" dirty_flag="$(blitz_git_dirty_flag)" started_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" temp_context="${context_file}.tmp.$$" temp_info="${run_dir}/run-info.json.tmp.$$" mkdir -p "${run_dir}" printf '%s\n' "${run_id}" > "${id_file}" cat > "${temp_context}" </dev/null || echo 0)" if (( size < max_bytes )); then return 0 fi for (( index=max_files; index>=1; index-- )); do if [[ "${index}" -eq "${max_files}" ]]; then rm -f "${path}.${index}" fi if [[ -f "${path}.${index}" ]]; then mv -f "${path}.${index}" "${path}.$(( index + 1 ))" fi done mv -f "${path}" "${path}.1" } blitz_jsonl_append_line() { local path="$1" local line="$2" mkdir -p "$(dirname "${path}")" blitz_jsonl_rotate_if_needed "${path}" printf '%s\n' "${line}" >> "${path}" } blitz_launch_incident_capture() { local launch_script="${BOOT_SCRIPT_DIR}/blitz-incident-capture-launch.sh" if [[ ! -f "${launch_script}" ]]; then return 1 fi "${launch_script}" "$@" >/dev/null 2>&1 || return 1 }