Release 260111

system/athena/athenad.py

@@ -0,0 +1,842 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+import os
+import queue
+import random
+import select
+import socket
+import sys
+import tempfile
+import threading
+import time
+from dataclasses import asdict, dataclass, replace
+from datetime import datetime
+from functools import partial, total_ordering
+from queue import Queue
+from typing import cast
+from collections.abc import Callable
+
+import requests
+from jsonrpc import JSONRPCResponseManager, dispatcher
+from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException,
+                       create_connection)
+
+import cereal.messaging as messaging
+from cereal import log
+from cereal.services import SERVICE_LIST
+from openpilot.common.api import Api
+from openpilot.common.file_helpers import CallbackReader, get_upload_stream
+from openpilot.common.params import Params
+from openpilot.common.realtime import set_core_affinity
+from openpilot.system.hardware import HARDWARE, PC
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.version import get_build_metadata
+from openpilot.system.hardware.hw import Paths
+
+
+ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai')
+HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4"))
+LOCAL_PORT_WHITELIST = {22, }  # SSH
+
+LOG_ATTR_NAME = 'user.upload'
+LOG_ATTR_VALUE_MAX_UNIX_TIME = int.to_bytes(2147483647, 4, sys.byteorder)
+RECONNECT_TIMEOUT_S = 70
+
+RETRY_DELAY = 10  # seconds
+MAX_RETRY_COUNT = 30  # Try for at most 5 minutes if upload fails immediately
+MAX_AGE = 31 * 24 * 3600  # seconds
+WS_FRAME_SIZE = 4096
+DEVICE_STATE_UPDATE_INTERVAL = 1.0  # in seconds
+DEFAULT_UPLOAD_PRIORITY = 99  # higher number = lower priority
+
+NetworkType = log.DeviceState.NetworkType
+
+UploadFileDict = dict[str, str | int | float | bool]
+UploadItemDict = dict[str, str | bool | int | float | dict[str, str]]
+
+UploadFilesToUrlResponse = dict[str, int | list[UploadItemDict] | list[str]]
+
+
+@dataclass
+class UploadFile:
+  fn: str
+  url: str
+  headers: dict[str, str]
+  allow_cellular: bool
+  priority: int = DEFAULT_UPLOAD_PRIORITY
+
+  @classmethod
+  def from_dict(cls, d: dict) -> UploadFile:
+    return cls(d.get("fn", ""), d.get("url", ""), d.get("headers", {}), d.get("allow_cellular", False), d.get("priority", DEFAULT_UPLOAD_PRIORITY))
+
+
+@dataclass
+@total_ordering
+class UploadItem:
+  path: str
+  url: str
+  headers: dict[str, str]
+  created_at: int
+  id: str | None
+  retry_count: int = 0
+  current: bool = False
+  progress: float = 0
+  allow_cellular: bool = False
+  priority: int = DEFAULT_UPLOAD_PRIORITY
+
+  @classmethod
+  def from_dict(cls, d: dict) -> UploadItem:
+    return cls(d["path"], d["url"], d["headers"], d["created_at"], d["id"], d["retry_count"], d["current"],
+               d["progress"], d["allow_cellular"], d["priority"])
+
+  def __lt__(self, other):
+    if not isinstance(other, UploadItem):
+      return NotImplemented
+    return self.priority < other.priority
+
+  def __eq__(self, other):
+    if not isinstance(other, UploadItem):
+      return NotImplemented
+    return self.priority == other.priority
+
+
+dispatcher["echo"] = lambda s: s
+recv_queue: Queue[str] = queue.Queue()
+send_queue: Queue[str] = queue.Queue()
+upload_queue: Queue[UploadItem] = queue.PriorityQueue()
+low_priority_send_queue: Queue[str] = queue.Queue()
+log_recv_queue: Queue[str] = queue.Queue()
+cancelled_uploads: set[str] = set()
+
+cur_upload_items: dict[int, UploadItem | None] = {}
+
+
+def strip_zst_extension(fn: str) -> str:
+  if fn.endswith('.zst'):
+    return fn[:-4]
+  return fn
+
+
+class AbortTransferException(Exception):
+  pass
+
+
+class UploadQueueCache:
+
+  @staticmethod
+  def initialize(upload_queue: Queue[UploadItem]) -> None:
+    try:
+      upload_queue_json = Params().get("AthenadUploadQueue")
+      if upload_queue_json is not None:
+        for item in json.loads(upload_queue_json):
+          upload_queue.put(UploadItem.from_dict(item))
+    except Exception:
+      cloudlog.exception("athena.UploadQueueCache.initialize.exception")
+
+  @staticmethod
+  def cache(upload_queue: Queue[UploadItem]) -> None:
+    try:
+      queue: list[UploadItem | None] = list(upload_queue.queue)
+      items = [asdict(i) for i in queue if i is not None and (i.id not in cancelled_uploads)]
+      Params().put("AthenadUploadQueue", json.dumps(items))
+    except Exception:
+      cloudlog.exception("athena.UploadQueueCache.cache.exception")
+
+
+def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
+  end_event = threading.Event()
+
+  threads = [
+    threading.Thread(target=ws_manage, args=(ws, end_event), name='ws_manage'),
+    threading.Thread(target=ws_recv, args=(ws, end_event), name='ws_recv'),
+    threading.Thread(target=ws_send, args=(ws, end_event), name='ws_send'),
+    threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler'),
+    threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler2'),
+    threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler3'),
+    threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler4'),
+    threading.Thread(target=log_handler, args=(end_event,), name='log_handler'),
+    threading.Thread(target=stat_handler, args=(end_event,), name='stat_handler'),
+  ] + [
+    threading.Thread(target=jsonrpc_handler, args=(end_event,), name=f'worker_{x}')
+    for x in range(HANDLER_THREADS)
+  ]
+
+  for thread in threads:
+    thread.start()
+  try:
+    while not end_event.wait(0.1):
+      if exit_event is not None and exit_event.is_set():
+        end_event.set()
+  except (KeyboardInterrupt, SystemExit):
+    end_event.set()
+    raise
+  finally:
+    for thread in threads:
+      cloudlog.debug(f"athena.joining {thread.name}")
+      thread.join()
+
+
+def jsonrpc_handler(end_event: threading.Event) -> None:
+  dispatcher["startLocalProxy"] = partial(startLocalProxy, end_event)
+  while not end_event.is_set():
+    try:
+      data = recv_queue.get(timeout=1)
+      if "method" in data:
+        cloudlog.event("athena.jsonrpc_handler.call_method", data=data)
+        response = JSONRPCResponseManager.handle(data, dispatcher)
+        send_queue.put_nowait(response.json)
+      elif "id" in data and ("result" in data or "error" in data):
+        log_recv_queue.put_nowait(data)
+      else:
+        raise Exception("not a valid request or response")
+    except queue.Empty:
+      pass
+    except Exception as e:
+      cloudlog.exception("athena jsonrpc handler failed")
+      send_queue.put_nowait(json.dumps({"error": str(e)}))
+
+
+def retry_upload(tid: int, end_event: threading.Event, increase_count: bool = True) -> None:
+  item = cur_upload_items[tid]
+  if item is not None and item.retry_count < MAX_RETRY_COUNT:
+    new_retry_count = item.retry_count + 1 if increase_count else item.retry_count
+
+    item = replace(
+      item,
+      retry_count=new_retry_count,
+      progress=0,
+      current=False
+    )
+    upload_queue.put_nowait(item)
+    UploadQueueCache.cache(upload_queue)
+
+    cur_upload_items[tid] = None
+
+    for _ in range(RETRY_DELAY):
+      time.sleep(1)
+      if end_event.is_set():
+        break
+
+
+def cb(sm, item, tid, end_event: threading.Event, sz: int, cur: int) -> None:
+  # Abort transfer if connection changed to metered after starting upload
+  # or if athenad is shutting down to re-connect the websocket
+  if not item.allow_cellular:
+    if (time.monotonic() - sm.recv_time['deviceState']) > DEVICE_STATE_UPDATE_INTERVAL:
+      sm.update(0)
+      if sm['deviceState'].networkMetered:
+        raise AbortTransferException
+
+  if end_event.is_set():
+    raise AbortTransferException
+
+  cur_upload_items[tid] = replace(item, progress=cur / sz if sz else 1)
+
+
+def upload_handler(end_event: threading.Event) -> None:
+  sm = messaging.SubMaster(['deviceState'])
+  tid = threading.get_ident()
+
+  while not end_event.is_set():
+    cur_upload_items[tid] = None
+
+    try:
+      cur_upload_items[tid] = item = replace(upload_queue.get(timeout=1), current=True)
+
+      if item.id in cancelled_uploads:
+        cancelled_uploads.remove(item.id)
+        continue
+
+      # Remove item if too old
+      age = datetime.now() - datetime.fromtimestamp(item.created_at / 1000)
+      if age.total_seconds() > MAX_AGE:
+        cloudlog.event("athena.upload_handler.expired", item=item, error=True)
+        continue
+
+      # Check if uploading over metered connection is allowed
+      sm.update(0)
+      metered = sm['deviceState'].networkMetered
+      network_type = sm['deviceState'].networkType.raw
+      if metered and (not item.allow_cellular):
+        retry_upload(tid, end_event, False)
+        continue
+
+      try:
+        fn = item.path
+        try:
+          sz = os.path.getsize(fn)
+        except OSError:
+          sz = -1
+
+        cloudlog.event("athena.upload_handler.upload_start", fn=fn, sz=sz, network_type=network_type, metered=metered, retry_count=item.retry_count)
+
+        with _do_upload(item, partial(cb, sm, item, tid, end_event)) as response:
+          if response.status_code not in (200, 201, 401, 403, 412):
+            cloudlog.event("athena.upload_handler.retry", status_code=response.status_code, fn=fn, sz=sz, network_type=network_type, metered=metered)
+            retry_upload(tid, end_event)
+          else:
+            cloudlog.event("athena.upload_handler.success", fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+        UploadQueueCache.cache(upload_queue)
+      except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.SSLError):
+        cloudlog.event("athena.upload_handler.timeout", fn=fn, sz=sz, network_type=network_type, metered=metered)
+        retry_upload(tid, end_event)
+      except AbortTransferException:
+        cloudlog.event("athena.upload_handler.abort", fn=fn, sz=sz, network_type=network_type, metered=metered)
+        retry_upload(tid, end_event, False)
+
+    except queue.Empty:
+      pass
+    except Exception:
+      cloudlog.exception("athena.upload_handler.exception")
+
+
+def _do_upload(upload_item: UploadItem, callback: Callable = None) -> requests.Response:
+  path = upload_item.path
+  compress = False
+
+  # If file does not exist, but does exist without the .zst extension we will compress on the fly
+  if not os.path.exists(path) and os.path.exists(strip_zst_extension(path)):
+    path = strip_zst_extension(path)
+    compress = True
+
+  stream = None
+  try:
+    stream, content_length = get_upload_stream(path, compress)
+    response = requests.put(upload_item.url,
+                            data=CallbackReader(stream, callback, content_length) if callback else stream,
+                            headers={**upload_item.headers, 'Content-Length': str(content_length)},
+                            timeout=30)
+    return response
+  finally:
+    if stream:
+      stream.close()
+
+
+# security: user should be able to request any message from their car
+@dispatcher.add_method
+def getMessage(service: str, timeout: int = 1000) -> dict:
+  if service is None or service not in SERVICE_LIST:
+    raise Exception("invalid service")
+
+  socket = messaging.sub_sock(service, timeout=timeout)
+  try:
+    ret = messaging.recv_one(socket)
+
+    if ret is None:
+      raise TimeoutError
+
+    # this is because capnp._DynamicStructReader doesn't have typing information
+    return cast(dict, ret.to_dict())
+  finally:
+    del socket
+
+
+@dispatcher.add_method
+def getVersion() -> dict[str, str]:
+  build_metadata = get_build_metadata()
+  return {
+    "version": build_metadata.openpilot.version,
+    "remote": build_metadata.openpilot.git_normalized_origin,
+    "branch": build_metadata.channel,
+    "commit": build_metadata.openpilot.git_commit,
+  }
+
+
+def scan_dir(path: str, prefix: str) -> list[str]:
+  files = []
+  # only walk directories that match the prefix
+  # (glob and friends traverse entire dir tree)
+  with os.scandir(path) as i:
+    for e in i:
+      rel_path = os.path.relpath(e.path, Paths.log_root())
+      if e.is_dir(follow_symlinks=False):
+        # add trailing slash
+        rel_path = os.path.join(rel_path, '')
+        # if prefix is a partial dir name, current dir will start with prefix
+        # if prefix is a partial file name, prefix with start with dir name
+        if rel_path.startswith(prefix) or prefix.startswith(rel_path):
+          files.extend(scan_dir(e.path, prefix))
+      else:
+        if rel_path.startswith(prefix):
+          files.append(rel_path)
+  return files
+
+@dispatcher.add_method
+def listDataDirectory(prefix='') -> list[str]:
+  return scan_dir(Paths.log_root(), prefix)
+
+
+@dispatcher.add_method
+def uploadFileToUrl(fn: str, url: str, headers: dict[str, str]) -> UploadFilesToUrlResponse:
+  # this is because mypy doesn't understand that the decorator doesn't change the return type
+  response: UploadFilesToUrlResponse = uploadFilesToUrls([{
+    "fn": fn,
+    "url": url,
+    "headers": headers,
+  }])
+  return response
+
+
+@dispatcher.add_method
+def uploadFilesToUrls(files_data: list[UploadFileDict]) -> UploadFilesToUrlResponse:
+  files = map(UploadFile.from_dict, files_data)
+
+  items: list[UploadItemDict] = []
+  failed: list[str] = []
+  for file in files:
+    if len(file.fn) == 0 or file.fn[0] == '/' or '..' in file.fn or len(file.url) == 0:
+      failed.append(file.fn)
+      continue
+
+    path = os.path.join(Paths.log_root(), file.fn)
+    if not os.path.exists(path) and not os.path.exists(strip_zst_extension(path)):
+      failed.append(file.fn)
+      continue
+
+    # Skip item if already in queue
+    url = file.url.split('?')[0]
+    if any(url == item['url'].split('?')[0] for item in listUploadQueue()):
+      continue
+
+    item = UploadItem(
+      path=path,
+      url=file.url,
+      headers=file.headers,
+      created_at=int(time.time() * 1000),
+      id=None,
+      allow_cellular=file.allow_cellular,
+      priority=file.priority,
+    )
+    upload_id = hashlib.sha1(str(item).encode()).hexdigest()
+    item = replace(item, id=upload_id)
+    upload_queue.put_nowait(item)
+    items.append(asdict(item))
+
+  UploadQueueCache.cache(upload_queue)
+
+  resp: UploadFilesToUrlResponse = {"enqueued": len(items), "items": items}
+  if failed:
+    cloudlog.event("athena.uploadFilesToUrls.failed", failed=failed, error=True)
+    resp["failed"] = failed
+
+  return resp
+
+
+@dispatcher.add_method
+def listUploadQueue() -> list[UploadItemDict]:
+  items = list(upload_queue.queue) + list(cur_upload_items.values())
+  return [asdict(i) for i in items if (i is not None) and (i.id not in cancelled_uploads)]
+
+
+@dispatcher.add_method
+def cancelUpload(upload_id: str | list[str]) -> dict[str, int | str]:
+  if not isinstance(upload_id, list):
+    upload_id = [upload_id]
+
+  uploading_ids = {item.id for item in list(upload_queue.queue)}
+  cancelled_ids = uploading_ids.intersection(upload_id)
+  if len(cancelled_ids) == 0:
+    return {"success": 0, "error": "not found"}
+
+  cancelled_uploads.update(cancelled_ids)
+  return {"success": 1}
+
+@dispatcher.add_method
+def setRouteViewed(route: str) -> dict[str, int | str]:
+  # maintain a list of the last 10 routes viewed in connect
+  params = Params()
+
+  r = params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
+  routes = [] if r is None else r.split(",")
+  routes.append(route)
+
+  # remove duplicates
+  routes = list(dict.fromkeys(routes))
+
+  params.put("AthenadRecentlyViewedRoutes", ",".join(routes[-10:]))
+  return {"success": 1}
+
+
+def startLocalProxy(global_end_event: threading.Event, remote_ws_uri: str, local_port: int) -> dict[str, int]:
+  try:
+    # migration, can be removed once 0.9.8 is out for a while
+    if local_port == 8022:
+      local_port = 22
+
+    if local_port not in LOCAL_PORT_WHITELIST:
+      raise Exception("Requested local port not whitelisted")
+
+    cloudlog.debug("athena.startLocalProxy.starting")
+
+    dongle_id = Params().get("DongleId").decode('utf8')
+    identity_token = Api(dongle_id).get_token()
+    ws = create_connection(remote_ws_uri,
+                           cookie="jwt=" + identity_token,
+                           enable_multithread=True)
+
+    # Set TOS to keep connection responsive while under load.
+    # DSCP of 36/HDD_LINUX_AC_VI with the minimum delay flag
+    ws.sock.setsockopt(socket.IPPROTO_IP, socket.IP_TOS, 0x90)
+
+    ssock, csock = socket.socketpair()
+    local_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    local_sock.connect(('127.0.0.1', local_port))
+    local_sock.setblocking(False)
+
+    proxy_end_event = threading.Event()
+    threads = [
+      threading.Thread(target=ws_proxy_recv, args=(ws, local_sock, ssock, proxy_end_event, global_end_event)),
+      threading.Thread(target=ws_proxy_send, args=(ws, local_sock, csock, proxy_end_event))
+    ]
+    for thread in threads:
+      thread.start()
+
+    cloudlog.debug("athena.startLocalProxy.started")
+    return {"success": 1}
+  except Exception as e:
+    cloudlog.exception("athenad.startLocalProxy.exception")
+    raise e
+
+
+@dispatcher.add_method
+def getPublicKey() -> str | None:
+  if not os.path.isfile(Paths.persist_root() + '/comma/id_rsa.pub'):
+    return None
+
+  with open(Paths.persist_root() + '/comma/id_rsa.pub') as f:
+    return f.read()
+
+
+@dispatcher.add_method
+def getSshAuthorizedKeys() -> str:
+  return Params().get("GithubSshKeys", encoding='utf8') or ''
+
+
+@dispatcher.add_method
+def getGithubUsername() -> str:
+  return Params().get("GithubUsername", encoding='utf8') or ''
+
+@dispatcher.add_method
+def getSimInfo():
+  return HARDWARE.get_sim_info()
+
+
+@dispatcher.add_method
+def getNetworkType():
+  return HARDWARE.get_network_type()
+
+
+@dispatcher.add_method
+def getNetworkMetered() -> bool:
+  network_type = HARDWARE.get_network_type()
+  return HARDWARE.get_network_metered(network_type)
+
+
+@dispatcher.add_method
+def getNetworks():
+  return HARDWARE.get_networks()
+
+
+@dispatcher.add_method
+def takeSnapshot() -> str | dict[str, str] | None:
+  from openpilot.system.camerad.snapshot.snapshot import jpeg_write, snapshot
+  ret = snapshot()
+  if ret is not None:
+    def b64jpeg(x):
+      if x is not None:
+        f = io.BytesIO()
+        jpeg_write(f, x)
+        return base64.b64encode(f.getvalue()).decode("utf-8")
+      else:
+        return None
+    return {'jpegBack': b64jpeg(ret[0]),
+            'jpegFront': b64jpeg(ret[1])}
+  else:
+    raise Exception("not available while camerad is started")
+
+
+def get_logs_to_send_sorted() -> list[str]:
+  # TODO: scan once then use inotify to detect file creation/deletion
+  curr_time = int(time.time())
+  logs = []
+  for log_entry in os.listdir(Paths.swaglog_root()):
+    log_path = os.path.join(Paths.swaglog_root(), log_entry)
+    time_sent = 0
+    try:
+      value = getxattr(log_path, LOG_ATTR_NAME)
+      if value is not None:
+        time_sent = int.from_bytes(value, sys.byteorder)
+    except (ValueError, TypeError):
+      pass
+    # assume send failed and we lost the response if sent more than one hour ago
+    if not time_sent or curr_time - time_sent > 3600:
+      logs.append(log_entry)
+  # excluding most recent (active) log file
+  return sorted(logs)[:-1]
+
+
+def log_handler(end_event: threading.Event) -> None:
+  if PC:
+    return
+
+  log_files = []
+  last_scan = 0.
+  while not end_event.is_set():
+    try:
+      curr_scan = time.monotonic()
+      if curr_scan - last_scan > 10:
+        log_files = get_logs_to_send_sorted()
+        last_scan = curr_scan
+
+      # send one log
+      curr_log = None
+      if len(log_files) > 0:
+        log_entry = log_files.pop() # newest log file
+        cloudlog.debug(f"athena.log_handler.forward_request {log_entry}")
+        try:
+          curr_time = int(time.time())
+          log_path = os.path.join(Paths.swaglog_root(), log_entry)
+          setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder))
+          with open(log_path) as f:
+            jsonrpc = {
+              "method": "forwardLogs",
+              "params": {
+                "logs": f.read()
+              },
+              "jsonrpc": "2.0",
+              "id": log_entry
+            }
+            low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
+            curr_log = log_entry
+        except OSError:
+          pass  # file could be deleted by log rotation
+
+      # wait for response up to ~100 seconds
+      # always read queue at least once to process any old responses that arrive
+      for _ in range(100):
+        if end_event.is_set():
+          break
+        try:
+          log_resp = json.loads(log_recv_queue.get(timeout=1))
+          log_entry = log_resp.get("id")
+          log_success = "result" in log_resp and log_resp["result"].get("success")
+          cloudlog.debug(f"athena.log_handler.forward_response {log_entry} {log_success}")
+          if log_entry and log_success:
+            log_path = os.path.join(Paths.swaglog_root(), log_entry)
+            try:
+              setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME)
+            except OSError:
+              pass  # file could be deleted by log rotation
+          if curr_log == log_entry:
+            break
+        except queue.Empty:
+          if curr_log is None:
+            break
+
+    except Exception:
+      cloudlog.exception("athena.log_handler.exception")
+
+
+def stat_handler(end_event: threading.Event) -> None:
+  STATS_DIR = Paths.stats_root()
+  last_scan = 0.0
+
+  while not end_event.is_set():
+    curr_scan = time.monotonic()
+    try:
+      if curr_scan - last_scan > 10:
+        stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
+        if len(stat_filenames) > 0:
+          stat_path = os.path.join(STATS_DIR, stat_filenames[0])
+          with open(stat_path) as f:
+            jsonrpc = {
+              "method": "storeStats",
+              "params": {
+                "stats": f.read()
+              },
+              "jsonrpc": "2.0",
+              "id": stat_filenames[0]
+            }
+            low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
+          os.remove(stat_path)
+        last_scan = curr_scan
+    except Exception:
+      cloudlog.exception("athena.stat_handler.exception")
+    time.sleep(0.1)
+
+
+def ws_proxy_recv(ws: WebSocket, local_sock: socket.socket, ssock: socket.socket, end_event: threading.Event, global_end_event: threading.Event) -> None:
+  while not (end_event.is_set() or global_end_event.is_set()):
+    try:
+      r = select.select((ws.sock,), (), (), 30)
+      if r[0]:
+        data = ws.recv()
+        if isinstance(data, str):
+          data = data.encode("utf-8")
+        local_sock.sendall(data)
+    except WebSocketTimeoutException:
+      pass
+    except Exception:
+      cloudlog.exception("athenad.ws_proxy_recv.exception")
+      break
+
+  cloudlog.debug("athena.ws_proxy_recv closing sockets")
+  ssock.close()
+  local_sock.close()
+  ws.close()
+  cloudlog.debug("athena.ws_proxy_recv done closing sockets")
+
+  end_event.set()
+
+
+def ws_proxy_send(ws: WebSocket, local_sock: socket.socket, signal_sock: socket.socket, end_event: threading.Event) -> None:
+  while not end_event.is_set():
+    try:
+      r, _, _ = select.select((local_sock, signal_sock), (), ())
+      if r:
+        if r[0].fileno() == signal_sock.fileno():
+          # got end signal from ws_proxy_recv
+          end_event.set()
+          break
+        data = local_sock.recv(4096)
+        if not data:
+          # local_sock is dead
+          end_event.set()
+          break
+
+        ws.send(data, ABNF.OPCODE_BINARY)
+    except Exception:
+      cloudlog.exception("athenad.ws_proxy_send.exception")
+      end_event.set()
+
+  cloudlog.debug("athena.ws_proxy_send closing sockets")
+  signal_sock.close()
+  cloudlog.debug("athena.ws_proxy_send done closing sockets")
+
+
+def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
+  last_ping = int(time.monotonic() * 1e9)
+  while not end_event.is_set():
+    try:
+      opcode, data = ws.recv_data(control_frame=True)
+      if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
+        if opcode == ABNF.OPCODE_TEXT:
+          data = data.decode("utf-8")
+        recv_queue.put_nowait(data)
+      elif opcode == ABNF.OPCODE_PING:
+        last_ping = int(time.monotonic() * 1e9)
+        Params().put("LastAthenaPingTime", str(last_ping))
+    except WebSocketTimeoutException:
+      ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping
+      if ns_since_last_ping > RECONNECT_TIMEOUT_S * 1e9:
+        cloudlog.exception("athenad.ws_recv.timeout")
+        end_event.set()
+    except Exception:
+      cloudlog.exception("athenad.ws_recv.exception")
+      end_event.set()
+
+
+def ws_send(ws: WebSocket, end_event: threading.Event) -> None:
+  while not end_event.is_set():
+    try:
+      try:
+        data = send_queue.get_nowait()
+      except queue.Empty:
+        data = low_priority_send_queue.get(timeout=1)
+      for i in range(0, len(data), WS_FRAME_SIZE):
+        frame = data[i:i+WS_FRAME_SIZE]
+        last = i + WS_FRAME_SIZE >= len(data)
+        opcode = ABNF.OPCODE_TEXT if i == 0 else ABNF.OPCODE_CONT
+        ws.send_frame(ABNF.create_frame(frame, opcode, last))
+    except queue.Empty:
+      pass
+    except Exception:
+      cloudlog.exception("athenad.ws_send.exception")
+      end_event.set()
+
+
+def ws_manage(ws: WebSocket, end_event: threading.Event) -> None:
+  params = Params()
+  onroad_prev = None
+  sock = ws.sock
+
+  while True:
+    onroad = params.get_bool("IsOnroad")
+    if onroad != onroad_prev:
+      onroad_prev = onroad
+
+      if sock is not None:
+        # While not sending data, onroad, we can expect to time out in 7 + (7 * 2) = 21s
+        #                         offroad, we can expect to time out in 30 + (10 * 3) = 60s
+        # FIXME: TCP_USER_TIMEOUT is effectively 2x for some reason (32s), so it's mostly unused
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, 16000 if onroad else 0)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7 if onroad else 30)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 7 if onroad else 10)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 2 if onroad else 3)
+
+    if end_event.wait(5):
+      break
+
+
+def backoff(retries: int) -> int:
+  return random.randrange(0, min(128, int(2 ** retries)))
+
+
+def main(exit_event: threading.Event = None):
+  try:
+    set_core_affinity([0, 1, 2, 3])
+  except Exception:
+    cloudlog.exception("failed to set core affinity")
+
+  params = Params()
+  dongle_id = params.get("DongleId", encoding='utf-8')
+  UploadQueueCache.initialize(upload_queue)
+
+  ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
+  api = Api(dongle_id)
+
+  conn_start = None
+  conn_retries = 0
+  while exit_event is None or not exit_event.is_set():
+    try:
+      if conn_start is None:
+        conn_start = time.monotonic()
+
+      cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri, retries=conn_retries)
+      ws = create_connection(ws_uri,
+                             cookie="jwt=" + api.get_token(),
+                             enable_multithread=True,
+                             timeout=30.0)
+      cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri, retries=conn_retries,
+                     duration=time.monotonic() - conn_start)
+      conn_start = None
+
+      conn_retries = 0
+      cur_upload_items.clear()
+
+      handle_long_poll(ws, exit_event)
+
+      ws.close()
+    except (KeyboardInterrupt, SystemExit):
+      break
+    except (ConnectionError, TimeoutError, WebSocketException):
+      conn_retries += 1
+      params.remove("LastAthenaPingTime")
+    except Exception:
+      cloudlog.exception("athenad.main.exception")
+
+      conn_retries += 1
+      params.remove("LastAthenaPingTime")
+
+    time.sleep(backoff(conn_retries))
+
+
+if __name__ == "__main__":
+  main()

system/athena/manage_athenad.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import time
+from multiprocessing import Process
+
+from openpilot.common.params import Params
+from openpilot.system.manager.process import launcher
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware import HARDWARE
+from openpilot.system.version import get_build_metadata
+
+ATHENA_MGR_PID_PARAM = "AthenadPid"
+
+
+def main():
+  params = Params()
+  dongle_id = params.get("DongleId").decode('utf-8')
+  build_metadata = get_build_metadata()
+
+  cloudlog.bind_global(dongle_id=dongle_id,
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
+                       device=HARDWARE.get_device_type())
+
+  try:
+    while 1:
+      cloudlog.info("starting athena daemon")
+      proc = Process(name='athenad', target=launcher, args=('system.athena.athenad', 'athenad'))
+      proc.start()
+      proc.join()
+      cloudlog.event("athenad exited", exitcode=proc.exitcode)
+      time.sleep(5)
+  except Exception:
+    cloudlog.exception("manage_athenad.exception")
+  finally:
+    params.remove(ATHENA_MGR_PID_PARAM)
+
+
+if __name__ == '__main__':
+  main()

system/athena/registration.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+import time
+import json
+import jwt
+from pathlib import Path
+
+from datetime import datetime, timedelta, UTC
+from openpilot.common.api import api_get
+from openpilot.common.params import Params
+from openpilot.common.spinner import Spinner
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from openpilot.system.hardware import HARDWARE, PC
+from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+
+
+UNREGISTERED_DONGLE_ID = "UnregisteredDevice"
+
+DUMMY_IMEI1 = '865420071781912'
+DUMMY_IMEI2 = '865420071781904'
+
+
+def is_registered_device() -> bool:
+  dongle = Params().get("DongleId", encoding='utf-8')
+  return dongle not in (None, UNREGISTERED_DONGLE_ID)
+
+
+def register(show_spinner=False) -> str | None:
+  """
+  All devices built since March 2024 come with all
+  info stored in /persist/. This is kept around
+  only for devices built before then.
+
+  With a backend update to take serial number instead
+  of dongle ID to some endpoints, this can be removed
+  entirely.
+  """
+  params = Params()
+
+
+  #return UNREGISTERED_DONGLE_ID  # for c3lite, clone
+  dongle_id: str | None = params.get("DongleId", encoding='utf8')
+  if dongle_id is None and Path(Paths.persist_root()+"/comma/dongle_id").is_file():
+    # not all devices will have this; added early in comma 3X production (2/28/24)
+    with open(Paths.persist_root()+"/comma/dongle_id") as f:
+      dongle_id = f.read().strip()
+
+  pubkey = Path(Paths.persist_root()+"/comma/id_rsa.pub")
+  if not pubkey.is_file():
+    dongle_id = UNREGISTERED_DONGLE_ID
+    cloudlog.warning(f"missing public key: {pubkey}")
+  elif dongle_id is None:
+    if show_spinner:
+      spinner = Spinner()
+      spinner.update("registering device")
+
+    # Create registration token, in the future, this key will make JWTs directly
+    with open(Paths.persist_root()+"/comma/id_rsa.pub") as f1, open(Paths.persist_root()+"/comma/id_rsa") as f2:
+      public_key = f1.read()
+      private_key = f2.read()
+
+    # Block until we get the imei
+    serial = HARDWARE.get_serial()
+    start_time = time.monotonic()
+    imei1: str | None = None
+    imei2: str | None = None
+
+    while imei1 is None and imei2 is None:
+      try:
+        imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
+      except Exception:
+        cloudlog.exception("Error getting imei, trying again...")
+        time.sleep(1)
+
+      if time.monotonic() - start_time > 30 and show_spinner:
+        spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
+        imei1 = DUMMY_IMEI1
+        imei2 = DUMMY_IMEI2
+        break
+
+    backoff = 0
+    start_time = time.monotonic()
+    while True:
+      try:
+        register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)}, private_key, algorithm='RS256')
+        cloudlog.info("getting pilotauth")
+        resp = api_get("v2/pilotauth/", method='POST', timeout=15,
+                       imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token)
+
+        if resp.status_code in (402, 403):
+          cloudlog.info(f"Unable to register device, got {resp.status_code}")
+          dongle_id = UNREGISTERED_DONGLE_ID
+        else:
+          dongleauth = json.loads(resp.text)
+          dongle_id = dongleauth["dongle_id"]
+        break
+      except Exception:
+        cloudlog.exception("failed to authenticate")
+        backoff = min(backoff + 1, 15)
+        time.sleep(backoff)
+
+      if time.monotonic() - start_time > 14:
+        cloudlog.error("pilotauth timed out; continuing as UNREGISTERED")
+        dongle_id = UNREGISTERED_DONGLE_ID
+        break
+
+      if time.monotonic() - start_time > 60 and show_spinner:
+        spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
+
+    if show_spinner:
+      spinner.close()
+
+  if dongle_id:
+    params.put("DongleId", dongle_id)
+    #set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
+  return dongle_id
+
+
+if __name__ == "__main__":
+  print(register())

system/athena/tests/helpers.py

@@ -0,0 +1,70 @@
+import http.server
+import socket
+
+
+class MockResponse:
+  def __init__(self, json, status_code):
+    self.json = json
+    self.text = json
+    self.status_code = status_code
+
+
+class EchoSocket:
+  def __init__(self, port):
+    self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    self.socket.bind(('127.0.0.1', port))
+    self.socket.listen(1)
+
+  def run(self):
+    conn, _ = self.socket.accept()
+    conn.settimeout(5.0)
+
+    try:
+      while True:
+        data = conn.recv(4096)
+        if data:
+          print(f'EchoSocket got {data}')
+          conn.sendall(data)
+        else:
+          break
+    finally:
+      conn.shutdown(0)
+      conn.close()
+      self.socket.shutdown(0)
+      self.socket.close()
+
+
+class MockApi:
+  def __init__(self, dongle_id):
+    pass
+
+  def get_token(self):
+    return "fake-token"
+
+
+class MockWebsocket:
+  sock = socket.socket()
+
+  def __init__(self, recv_queue, send_queue):
+    self.recv_queue = recv_queue
+    self.send_queue = send_queue
+
+  def recv(self):
+    data = self.recv_queue.get()
+    if isinstance(data, Exception):
+      raise data
+    return data
+
+  def send(self, data, opcode):
+    self.send_queue.put_nowait((data, opcode))
+
+  def close(self):
+    pass
+
+
+class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+  def do_PUT(self):
+    length = int(self.headers['Content-Length'])
+    self.rfile.read(length)
+    self.send_response(201, "Created")
+    self.end_headers()

system/athena/tests/test_athenad.py

@@ -0,0 +1,428 @@
+import pytest
+from functools import wraps
+import json
+import multiprocessing
+import os
+import requests
+import shutil
+import time
+import threading
+import queue
+from dataclasses import asdict, replace
+from datetime import datetime, timedelta
+
+from websocket import ABNF
+from websocket._exceptions import WebSocketConnectionClosedException
+
+from cereal import messaging
+
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.athena import athenad
+from openpilot.system.athena.athenad import MAX_RETRY_COUNT, dispatcher
+from openpilot.system.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket
+from openpilot.selfdrive.test.helpers import http_server_context
+from openpilot.system.hardware.hw import Paths
+
+
+def seed_athena_server(host, port):
+  with Timeout(2, 'HTTP Server seeding failed'):
+    while True:
+      try:
+        requests.put(f'http://{host}:{port}/qlog.zst', data='', timeout=10)
+        break
+      except requests.exceptions.ConnectionError:
+        time.sleep(0.1)
+
+def with_upload_handler(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    end_event = threading.Event()
+    thread = threading.Thread(target=athenad.upload_handler, args=(end_event,))
+    thread.start()
+    try:
+      return func(*args, **kwargs)
+    finally:
+      end_event.set()
+      thread.join()
+  return wrapper
+
+@pytest.fixture
+def mock_create_connection(mocker):
+    return mocker.patch('openpilot.system.athena.athenad.create_connection')
+
+@pytest.fixture
+def host():
+  with http_server_context(handler=HTTPRequestHandler, setup=seed_athena_server) as (host, port):
+    yield f"http://{host}:{port}"
+
+class TestAthenadMethods:
+  @classmethod
+  def setup_class(cls):
+    cls.SOCKET_PORT = 45454
+    athenad.Api = MockApi
+    athenad.LOCAL_PORT_WHITELIST = {cls.SOCKET_PORT}
+
+  def setup_method(self):
+    self.default_params = {
+      "DongleId": "0000000000000000",
+      "GithubSshKeys": b"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501
+      "GithubUsername": b"commaci",
+      "AthenadUploadQueue": '[]',
+    }
+
+    self.params = Params()
+    for k, v in self.default_params.items():
+      self.params.put(k, v)
+    self.params.put_bool("GsmMetered", True)
+
+    athenad.upload_queue = queue.Queue()
+    athenad.cur_upload_items.clear()
+    athenad.cancelled_uploads.clear()
+
+    for i in os.listdir(Paths.log_root()):
+      p = os.path.join(Paths.log_root(), i)
+      if os.path.isdir(p):
+        shutil.rmtree(p)
+      else:
+        os.unlink(p)
+
+  # *** test helpers ***
+
+  @staticmethod
+  def _wait_for_upload():
+    now = time.time()
+    while time.time() - now < 5:
+      if athenad.upload_queue.qsize() == 0:
+        break
+
+  @staticmethod
+  def _create_file(file: str, parent: str = None, data: bytes = b'') -> str:
+    fn = os.path.join(Paths.log_root() if parent is None else parent, file)
+    os.makedirs(os.path.dirname(fn), exist_ok=True)
+    with open(fn, 'wb') as f:
+      f.write(data)
+    return fn
+
+
+  # *** test cases ***
+
+  def test_echo(self):
+    assert dispatcher["echo"]("bob") == "bob"
+
+  def test_get_message(self):
+    with pytest.raises(TimeoutError) as _:
+      dispatcher["getMessage"]("controlsState")
+
+    end_event = multiprocessing.Event()
+
+    pub_sock = messaging.pub_sock("deviceState")
+
+    def send_deviceState():
+      while not end_event.is_set():
+        msg = messaging.new_message('deviceState')
+        pub_sock.send(msg.to_bytes())
+        time.sleep(0.01)
+
+    p = multiprocessing.Process(target=send_deviceState)
+    p.start()
+    time.sleep(0.1)
+    try:
+      deviceState = dispatcher["getMessage"]("deviceState")
+      assert deviceState['deviceState']
+    finally:
+      end_event.set()
+      p.join()
+
+  def test_list_data_directory(self):
+    route = '2021-03-29--13-32-47'
+    segments = [0, 1, 2, 3, 11]
+
+    filenames = ['qlog.zst', 'qcamera.ts', 'rlog.zst', 'fcamera.hevc', 'ecamera.hevc', 'dcamera.hevc']
+    files = [f'{route}--{s}/{f}' for s in segments for f in filenames]
+    for file in files:
+      self._create_file(file)
+
+    resp = dispatcher["listDataDirectory"]()
+    assert resp, 'list empty!'
+    assert len(resp) == len(files)
+
+    resp = dispatcher["listDataDirectory"](f'{route}--123')
+    assert len(resp) == 0
+
+    prefix = f'{route}'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1/'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1/q'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+  def test_strip_extension(self):
+    # any requested log file with an invalid extension won't return as existing
+    fn = self._create_file('qlog.bz2')
+    if fn.endswith('.bz2'):
+      assert athenad.strip_zst_extension(fn) == fn
+
+    fn = self._create_file('qlog.zst')
+    if fn.endswith('.zst'):
+      assert athenad.strip_zst_extension(fn) == fn[:-4]
+
+  @pytest.mark.parametrize("compress", [True, False])
+  def test_do_upload(self, host, compress):
+    # random bytes to ensure rather large object post-compression
+    fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
+
+    upload_fn = fn + ('.zst' if compress else '')
+    item = athenad.UploadItem(path=upload_fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')
+    with pytest.raises(requests.exceptions.ConnectionError):
+      athenad._do_upload(item)
+
+    item = athenad.UploadItem(path=upload_fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='')
+    resp = athenad._do_upload(item)
+    assert resp.status_code == 201
+
+  def test_upload_file_to_url(self, host):
+    fn = self._create_file('qlog.zst')
+
+    resp = dispatcher["uploadFileToUrl"]("qlog.zst", f"{host}/qlog.zst", {})
+    assert resp['enqueued'] == 1
+    assert 'failed' not in resp
+    assert {"path": fn, "url": f"{host}/qlog.zst", "headers": {}}.items() <= resp['items'][0].items()
+    assert resp['items'][0].get('id') is not None
+    assert athenad.upload_queue.qsize() == 1
+
+  def test_upload_file_to_url_duplicate(self, host):
+    self._create_file('qlog.zst')
+
+    url1 = f"{host}/qlog.zst?sig=sig1"
+    dispatcher["uploadFileToUrl"]("qlog.zst", url1, {})
+
+    # Upload same file again, but with different signature
+    url2 = f"{host}/qlog.zst?sig=sig2"
+    resp = dispatcher["uploadFileToUrl"]("qlog.zst", url2, {})
+    assert resp == {'enqueued': 0, 'items': []}
+
+  def test_upload_file_to_url_does_not_exist(self, host):
+    not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.zst", "http://localhost:1238", {})
+    assert not_exists_resp == {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.zst']}
+
+  @with_upload_handler
+  def test_upload_handler(self, host):
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # TODO: verify that upload actually succeeded
+    # TODO: also check that end_event and metered network raises AbortTransferException
+    assert athenad.upload_queue.qsize() == 0
+
+  @pytest.mark.parametrize("status,retry", [(500,True), (412,False)])
+  @with_upload_handler
+  def test_upload_handler_retry(self, mocker, host, status, retry):
+    mock_put = mocker.patch('requests.put')
+    mock_put.return_value.__enter__.return_value.status_code = status
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == (1 if retry else 0)
+
+    if retry:
+      assert athenad.upload_queue.get().retry_count == 1
+
+  @with_upload_handler
+  def test_upload_handler_timeout(self):
+    """When an upload times out or fails to connect it should be placed back in the queue"""
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
+    item_no_retry = replace(item, retry_count=MAX_RETRY_COUNT)
+
+    athenad.upload_queue.put_nowait(item_no_retry)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # Check that upload with retry count exceeded is not put back
+    assert athenad.upload_queue.qsize() == 0
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # Check that upload item was put back in the queue with incremented retry count
+    assert athenad.upload_queue.qsize() == 1
+    assert athenad.upload_queue.get().retry_count == 1
+
+  @with_upload_handler
+  def test_cancel_upload(self):
+    item = athenad.UploadItem(path="qlog.zst", url="http://localhost:44444/qlog.zst", headers={},
+                              created_at=int(time.time()*1000), id='id', allow_cellular=True)
+    athenad.upload_queue.put_nowait(item)
+    dispatcher["cancelUpload"](item.id)
+
+    assert item.id in athenad.cancelled_uploads
+
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == 0
+    assert len(athenad.cancelled_uploads) == 0
+
+  @with_upload_handler
+  def test_cancel_expiry(self):
+    t_future = datetime.now() - timedelta(days=40)
+    ts = int(t_future.strftime("%s")) * 1000
+
+    # Item that would time out if actually uploaded
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.zst", headers={}, created_at=ts, id='', allow_cellular=True)
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == 0
+
+  def test_list_upload_queue_empty(self):
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 0
+
+  @with_upload_handler
+  def test_list_upload_queue_current(self, host: str):
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 1
+    assert items[0]['current']
+
+  def test_list_upload_queue(self):
+    item = athenad.UploadItem(path="qlog.zst", url="http://localhost:44444/qlog.zst", headers={},
+                              created_at=int(time.time()*1000), id='id', allow_cellular=True)
+    athenad.upload_queue.put_nowait(item)
+
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 1
+    assert items[0] == asdict(item)
+    assert not items[0]['current']
+
+    athenad.cancelled_uploads.add(item.id)
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 0
+
+  def test_upload_queue_persistence(self):
+    item1 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id1')
+    item2 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id2')
+
+    athenad.upload_queue.put_nowait(item1)
+    athenad.upload_queue.put_nowait(item2)
+
+    # Ensure canceled items are not persisted
+    athenad.cancelled_uploads.add(item2.id)
+
+    # serialize item
+    athenad.UploadQueueCache.cache(athenad.upload_queue)
+
+    # deserialize item
+    athenad.upload_queue.queue.clear()
+    athenad.UploadQueueCache.initialize(athenad.upload_queue)
+
+    assert athenad.upload_queue.qsize() == 1
+    assert asdict(athenad.upload_queue.queue[-1]) == asdict(item1)
+
+  def test_start_local_proxy(self, mock_create_connection):
+    end_event = threading.Event()
+
+    ws_recv = queue.Queue()
+    ws_send = queue.Queue()
+    mock_ws = MockWebsocket(ws_recv, ws_send)
+    mock_create_connection.return_value = mock_ws
+
+    echo_socket = EchoSocket(self.SOCKET_PORT)
+    socket_thread = threading.Thread(target=echo_socket.run)
+    socket_thread.start()
+
+    athenad.startLocalProxy(end_event, 'ws://localhost:1234', self.SOCKET_PORT)
+
+    ws_recv.put_nowait(b'ping')
+    try:
+      recv = ws_send.get(timeout=5)
+      assert recv == (b'ping', ABNF.OPCODE_BINARY), recv
+    finally:
+      # signal websocket close to athenad.ws_proxy_recv
+      ws_recv.put_nowait(WebSocketConnectionClosedException())
+      socket_thread.join()
+
+  def test_get_ssh_authorized_keys(self):
+    keys = dispatcher["getSshAuthorizedKeys"]()
+    assert keys == self.default_params["GithubSshKeys"].decode('utf-8')
+
+  def test_get_github_username(self):
+    keys = dispatcher["getGithubUsername"]()
+    assert keys == self.default_params["GithubUsername"].decode('utf-8')
+
+  def test_get_version(self):
+    resp = dispatcher["getVersion"]()
+    keys = ["version", "remote", "branch", "commit"]
+    assert list(resp.keys()) == keys
+    for k in keys:
+      assert isinstance(resp[k], str), f"{k} is not a string"
+      assert len(resp[k]) > 0, f"{k} has no value"
+
+  def test_jsonrpc_handler(self):
+    end_event = threading.Event()
+    thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,))
+    thread.daemon = True
+    thread.start()
+    try:
+      # with params
+      athenad.recv_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0}))
+      resp = athenad.send_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}
+      # without params
+      athenad.recv_queue.put_nowait(json.dumps({"method": "getNetworkType", "jsonrpc": "2.0", "id": 0}))
+      resp = athenad.send_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': 1, 'id': 0, 'jsonrpc': '2.0'}
+      # log forwarding
+      athenad.recv_queue.put_nowait(json.dumps({'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}))
+      resp = athenad.log_recv_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}
+    finally:
+      end_event.set()
+      thread.join()
+
+  def test_get_logs_to_send_sorted(self):
+    fl = list()
+    for i in range(10):
+      file = f'swaglog.{i:010}'
+      self._create_file(file, Paths.swaglog_root())
+      fl.append(file)
+
+    # ensure the list is all logs except most recent
+    sl = athenad.get_logs_to_send_sorted()
+    assert sl == fl[:-1]

system/athena/tests/test_athenad_ping.py

@@ -0,0 +1,102 @@
+import pytest
+import subprocess
+import threading
+import time
+from typing import cast
+
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.athena import athenad
+from openpilot.system.manager.helpers import write_onroad_params
+from openpilot.system.hardware import TICI
+
+TIMEOUT_TOLERANCE = 20  # seconds
+
+
+def wifi_radio(on: bool) -> None:
+  if not TICI:
+    return
+  print(f"wifi {'on' if on else 'off'}")
+  subprocess.run(["nmcli", "radio", "wifi", "on" if on else "off"], check=True)
+
+
+class TestAthenadPing:
+  params: Params
+  dongle_id: str
+
+  athenad: threading.Thread
+  exit_event: threading.Event
+
+  def _get_ping_time(self) -> str | None:
+    return cast(str | None, self.params.get("LastAthenaPingTime", encoding="utf-8"))
+
+  def _clear_ping_time(self) -> None:
+    self.params.remove("LastAthenaPingTime")
+
+  def _received_ping(self) -> bool:
+    return self._get_ping_time() is not None
+
+  @classmethod
+  def teardown_class(cls) -> None:
+    wifi_radio(True)
+
+  def setup_method(self) -> None:
+    self.params = Params()
+    self.dongle_id = self.params.get("DongleId", encoding="utf-8")
+
+    wifi_radio(True)
+    self._clear_ping_time()
+
+    self.exit_event = threading.Event()
+    self.athenad = threading.Thread(target=athenad.main, args=(self.exit_event,))
+
+  def teardown_method(self) -> None:
+    if self.athenad.is_alive():
+      self.exit_event.set()
+      self.athenad.join()
+
+  def assertTimeout(self, reconnect_time: float, subtests, mocker) -> None:
+    self.athenad.start()
+
+    mock_create_connection = mocker.patch('openpilot.system.athena.athenad.create_connection',
+                                          new_callable=lambda: mocker.MagicMock(wraps=athenad.create_connection))
+
+    time.sleep(1)
+    mock_create_connection.assert_called_once()
+    mock_create_connection.reset_mock()
+
+    # check normal behavior, server pings on connection
+    with subtests.test("Wi-Fi: receives ping"), Timeout(70, "no ping received"):
+      while not self._received_ping():
+        time.sleep(0.1)
+      print("ping received")
+
+    mock_create_connection.assert_not_called()
+
+    # websocket should attempt reconnect after short time
+    with subtests.test("LTE: attempt reconnect"):
+      wifi_radio(False)
+      print("waiting for reconnect attempt")
+      start_time = time.monotonic()
+      with Timeout(reconnect_time, "no reconnect attempt"):
+        while not mock_create_connection.called:
+          time.sleep(0.1)
+        print(f"reconnect attempt after {time.monotonic() - start_time:.2f}s")
+
+    self._clear_ping_time()
+
+    # check ping received after reconnect
+    with subtests.test("LTE: receives ping"), Timeout(70, "no ping received"):
+      while not self._received_ping():
+        time.sleep(0.1)
+      print("ping received")
+
+  @pytest.mark.skipif(not TICI, reason="only run on desk")
+  def test_offroad(self, subtests, mocker) -> None:
+    write_onroad_params(False, self.params)
+    self.assertTimeout(60 + TIMEOUT_TOLERANCE, subtests, mocker)  # based using TCP keepalive settings
+
+  @pytest.mark.skipif(not TICI, reason="only run on desk")
+  def test_onroad(self, subtests, mocker) -> None:
+    write_onroad_params(True, self.params)
+    self.assertTimeout(21 + TIMEOUT_TOLERANCE, subtests, mocker)

system/athena/tests/test_registration.py

@@ -0,0 +1,76 @@
+import json
+from Crypto.PublicKey import RSA
+from pathlib import Path
+
+from openpilot.common.params import Params
+from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.system.athena.tests.helpers import MockResponse
+from openpilot.system.hardware.hw import Paths
+
+
+class TestRegistration:
+
+  def setup_method(self):
+    # clear params and setup key paths
+    self.params = Params()
+
+    persist_dir = Path(Paths.persist_root()) / "comma"
+    persist_dir.mkdir(parents=True, exist_ok=True)
+
+    self.priv_key = persist_dir / "id_rsa"
+    self.pub_key = persist_dir / "id_rsa.pub"
+    self.dongle_id = persist_dir / "dongle_id"
+
+  def _generate_keys(self):
+    self.pub_key.touch()
+    k = RSA.generate(2048)
+    with open(self.priv_key, "wb") as f:
+      f.write(k.export_key())
+    with open(self.pub_key, "wb") as f:
+      f.write(k.publickey().export_key())
+
+  def test_valid_cache(self, mocker):
+    # if all params are written, return the cached dongle id.
+    # should work with a dongle ID on either /persist/ or normal params
+    self._generate_keys()
+
+    dongle = "DONGLE_ID_123"
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    for persist, params in [(True, True), (True, False), (False, True)]:
+      self.params.put("DongleId", dongle if params else "")
+      with open(self.dongle_id, "w") as f:
+        f.write(dongle if persist else "")
+      assert register() == dongle
+      assert not m.called
+
+  def test_no_keys(self, mocker):
+    # missing pubkey
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    dongle = register()
+    assert m.call_count == 0
+    assert dongle == UNREGISTERED_DONGLE_ID
+    assert self.params.get("DongleId", encoding='utf-8') == dongle
+
+  def test_missing_cache(self, mocker):
+    # keys exist but no dongle id
+    self._generate_keys()
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    dongle = "DONGLE_ID_123"
+    m.return_value = MockResponse(json.dumps({'dongle_id': dongle}), 200)
+    assert register() == dongle
+    assert m.call_count == 1
+
+    # call again, shouldn't hit the API this time
+    assert register() == dongle
+    assert m.call_count == 1
+    assert self.params.get("DongleId", encoding='utf-8') == dongle
+
+  def test_unregistered(self, mocker):
+    # keys exist, but unregistered
+    self._generate_keys()
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    m.return_value = MockResponse(None, 402)
+    dongle = register()
+    assert m.call_count == 1
+    assert dongle == UNREGISTERED_DONGLE_ID
+    assert self.params.get("DongleId", encoding='utf-8') == dongle

system/camerad/cameras/camera_common.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <memory>
+
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionipc_server.h"
+#include "common/util.h"
+
+
+const int VIPC_BUFFER_COUNT = 18;
+
+typedef struct FrameMetadata {
+  uint32_t frame_id;
+  uint32_t request_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+  float processing_time;
+} FrameMetadata;
+
+class SpectraCamera;
+
+class CameraBuf {
+private:
+  int frame_buf_count;
+
+public:
+  VisionIpcServer *vipc_server;
+  VisionStreamType stream_type;
+
+  int cur_buf_idx;
+  FrameMetadata cur_frame_data;
+  VisionBuf *cur_yuv_buf;
+  VisionBuf *cur_camera_buf;
+  std::unique_ptr<VisionBuf[]> camera_bufs_raw;
+  uint32_t out_img_width, out_img_height;
+
+  CameraBuf() = default;
+  ~CameraBuf();
+  void init(cl_device_id device_id, cl_context context, SpectraCamera *cam, VisionIpcServer * v, int frame_cnt, VisionStreamType type);
+  void sendFrameToVipc();
+};
+
+void camerad_thread();
+kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b);
+float calculate_exposure_value(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip);
+int open_v4l_by_name_and_index(const char name[], int index = 0, int flags = O_RDWR | O_NONBLOCK);

system/camerad/cameras/cdm.h

@@ -0,0 +1,79 @@
+#pragma once
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <vector>
+#include <memory>
+
+// our helpers
+int write_random(uint8_t *dst, const std::vector<uint32_t> &vals);
+int write_cont(uint8_t *dst, uint32_t reg, const std::vector<uint32_t> &vals);
+int write_dmi(uint8_t *dst, uint64_t *addr, uint32_t length, uint32_t dmi_addr, uint8_t sel);
+
+// from drivers/media/platform/msm/camera/cam_cdm/cam_cdm_util.{c,h}
+
+enum cam_cdm_command {
+  CAM_CDM_CMD_UNUSED = 0x0,
+  CAM_CDM_CMD_DMI = 0x1,
+  CAM_CDM_CMD_NOT_DEFINED = 0x2,
+  CAM_CDM_CMD_REG_CONT = 0x3,
+  CAM_CDM_CMD_REG_RANDOM = 0x4,
+  CAM_CDM_CMD_BUFF_INDIRECT = 0x5,
+  CAM_CDM_CMD_GEN_IRQ = 0x6,
+  CAM_CDM_CMD_WAIT_EVENT = 0x7,
+  CAM_CDM_CMD_CHANGE_BASE = 0x8,
+  CAM_CDM_CMD_PERF_CTRL = 0x9,
+  CAM_CDM_CMD_DMI_32 = 0xa,
+  CAM_CDM_CMD_DMI_64 = 0xb,
+  CAM_CDM_CMD_PRIVATE_BASE = 0xc,
+  CAM_CDM_CMD_SWD_DMI_32 = (CAM_CDM_CMD_PRIVATE_BASE + 0x64),
+  CAM_CDM_CMD_SWD_DMI_64 = (CAM_CDM_CMD_PRIVATE_BASE + 0x65),
+  CAM_CDM_CMD_PRIVATE_BASE_MAX = 0x7F
+};
+
+/**
+ * struct cdm_regrandom_cmd - Definition for CDM random register command.
+ * @count: Number of register writes
+ * @reserved: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ */
+struct cdm_regrandom_cmd {
+  unsigned int count    : 16;
+  unsigned int reserved : 8;
+  unsigned int cmd      : 8;
+} __attribute__((__packed__));
+
+/**
+ * struct cdm_regcontinuous_cmd - Definition for a CDM register range command.
+ * @count: Number of register writes
+ * @reserved0: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ * @offset: Start address of the range of registers
+ * @reserved1: reserved bits
+ */
+struct cdm_regcontinuous_cmd {
+  unsigned int count     : 16;
+  unsigned int reserved0 : 8;
+  unsigned int cmd       : 8;
+  unsigned int offset    : 24;
+  unsigned int reserved1 : 8;
+} __attribute__((__packed__));
+
+/**
+ * struct cdm_dmi_cmd - Definition for a CDM DMI command.
+ * @length: Number of bytes in LUT - 1
+ * @reserved: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ * @addr: Address of the LUT in memory
+ * @DMIAddr: Address of the target DMI config register
+ * @DMISel: DMI identifier
+ */
+struct cdm_dmi_cmd {
+  unsigned int length   : 16;
+  unsigned int reserved : 8;
+  unsigned int cmd      : 8;
+  unsigned int addr;
+  unsigned int DMIAddr  : 24;
+  unsigned int DMISel   : 8;
+} __attribute__((__packed__));

system/camerad/cameras/hw.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include "common/util.h"
+#include "cereal/gen/cpp/log.capnp.h"
+#include "msgq/visionipc/visionipc_server.h"
+
+#include "media/cam_isp_ife.h"
+
+
+typedef enum {
+  ISP_RAW_OUTPUT,   // raw frame from sensor
+  ISP_IFE_PROCESSED,  // fully processed image through the IFE
+  ISP_BPS_PROCESSED,  // fully processed image through the BPS
+} SpectraOutputType;
+
+// For the comma 3X three camera platform
+
+struct CameraConfig {
+  int camera_num;
+  VisionStreamType stream_type;
+  float focal_len;  // millimeters
+  const char *publish_name;
+  cereal::FrameData::Builder (cereal::Event::Builder::*init_camera_state)();
+  bool enabled;
+  uint32_t phy;
+  bool vignetting_correction;
+  SpectraOutputType output_type;
+};
+
+// NOTE: to be able to disable road and wide road, we still have to configure the sensor over i2c
+// If you don't do this, the strobe GPIO is an output (even in reset it seems!)
+const CameraConfig WIDE_ROAD_CAMERA_CONFIG = {
+  .camera_num = 0,
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .focal_len = 1.71,
+  .publish_name = "wideRoadCameraState",
+  .init_camera_state = &cereal::Event::Builder::initWideRoadCameraState,
+  .enabled = !getenv("DISABLE_WIDE_ROAD"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_0,
+  .vignetting_correction = false,
+  .output_type = ISP_IFE_PROCESSED,
+};
+
+const CameraConfig ROAD_CAMERA_CONFIG = {
+  .camera_num = 1,
+  .stream_type = VISION_STREAM_ROAD,
+  .focal_len = 8.0,
+  .publish_name = "roadCameraState",
+  .init_camera_state = &cereal::Event::Builder::initRoadCameraState,
+  .enabled = !getenv("DISABLE_ROAD"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_1,
+  .vignetting_correction = true,
+  .output_type = ISP_IFE_PROCESSED,
+};
+
+const CameraConfig DRIVER_CAMERA_CONFIG = {
+  .camera_num = 2,
+  .stream_type = VISION_STREAM_DRIVER,
+  .focal_len = 1.71,
+  .publish_name = "driverCameraState",
+  .init_camera_state = &cereal::Event::Builder::initDriverCameraState,
+  .enabled = !getenv("DISABLE_DRIVER"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_2,
+  .vignetting_correction = false,
+  .output_type = ISP_BPS_PROCESSED,
+};
+
+const CameraConfig ALL_CAMERA_CONFIGS[] = {WIDE_ROAD_CAMERA_CONFIG, ROAD_CAMERA_CONFIG, DRIVER_CAMERA_CONFIG};

system/camerad/cameras/ife.h

@@ -0,0 +1,236 @@
+#pragma once
+
+#include "cdm.h"
+
+#include "system/camerad/cameras/hw.h"
+#include "system/camerad/sensors/sensor.h"
+
+int build_common_ife_bps(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches, bool ife) {
+  uint8_t *start = dst;
+
+  /*
+    Common between IFE and BPS.
+  */
+
+  // IFE -> BPS addresses
+  /*
+  std::map<uint32_t, uint32_t> addrs = {
+    {0xf30, 0x3468},
+  };
+  */
+
+  // YUV
+  dst += write_cont(dst, ife ? 0xf30 : 0x3468, {
+    0x00680208,
+    0x00000108,
+    0x00400000,
+    0x03ff0000,
+    0x01c01ed8,
+    0x00001f68,
+    0x02000000,
+    0x03ff0000,
+    0x1fb81e88,
+    0x000001c0,
+    0x02000000,
+    0x03ff0000,
+  });
+
+  return dst - start;
+}
+
+int build_update(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches) {
+  uint8_t *start = dst;
+
+  // init sequence
+  dst += write_random(dst, {
+    0x2c, 0xffffffff,
+    0x30, 0xffffffff,
+    0x34, 0xffffffff,
+    0x38, 0xffffffff,
+    0x3c, 0xffffffff,
+  });
+
+  // demux cfg
+  dst += write_cont(dst, 0x560, {
+    0x00000001,
+    0x04440444,
+    0x04450445,
+    0x04440444,
+    0x04450445,
+    0x000000ca,
+    0x0000009c,
+  });
+
+  // white balance
+  dst += write_cont(dst, 0x6fc, {
+    0x00800080,
+    0x00000080,
+    0x00000000,
+    0x00000000,
+  });
+
+  // module config/enables (e.g. enable debayer, white balance, etc.)
+  dst += write_cont(dst, 0x40, {
+    0x00000c06 | ((uint32_t)(cam.vignetting_correction) << 8),
+  });
+  dst += write_cont(dst, 0x44, {
+    0x00000000,
+  });
+  dst += write_cont(dst, 0x48, {
+    (1 << 3) | (1 << 1),
+  });
+  dst += write_cont(dst, 0x4c, {
+    0x00000019,
+  });
+  dst += write_cont(dst, 0xf00, {
+    0x00000000,
+  });
+
+  // cropping
+  dst += write_cont(dst, 0xe0c, {
+    0x00000e00,
+  });
+  dst += write_cont(dst, 0xe2c, {
+    0x00000e00,
+  });
+
+  // black level scale + offset
+  dst += write_cont(dst, 0x6b0, {
+    ((uint32_t)(1 << 11) << 0xf) | (s->black_level << (14 - s->bits_per_pixel)),
+    0x0,
+    0x0,
+  });
+
+  return dst - start;
+}
+
+
+int build_initial_config(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches, uint32_t out_width, uint32_t out_height) {
+  uint8_t *start = dst;
+
+  // start with the every frame config
+  dst += build_update(dst, cam, s, patches);
+
+  uint64_t addr;
+
+  // setup
+  dst += write_cont(dst, 0x478, {
+    0x00000004,
+    0x004000c0,
+  });
+  dst += write_cont(dst, 0x488, {
+    0x00000000,
+    0x00000000,
+    0x00000f0f,
+  });
+  dst += write_cont(dst, 0x49c, {
+    0x00000001,
+  });
+  dst += write_cont(dst, 0xce4, {
+    0x00000000,
+    0x00000000,
+  });
+
+  // linearization
+  dst += write_cont(dst, 0x4dc, {
+    0x00000000,
+  });
+  dst += write_cont(dst, 0x4e0, s->linearization_pts);
+  dst += write_cont(dst, 0x4f0, s->linearization_pts);
+  dst += write_cont(dst, 0x500, s->linearization_pts);
+  dst += write_cont(dst, 0x510, s->linearization_pts);
+  // TODO: this is DMI64 in the dump, does that matter?
+  dst += write_dmi(dst, &addr, s->linearization_lut.size()*sizeof(uint32_t), 0xc24, 9);
+  patches.push_back(addr - (uint64_t)start);
+
+  // vignetting correction
+  dst += write_cont(dst, 0x6bc, {
+    0x0b3c0000,
+    0x00670067,
+    0xd3b1300c,
+    0x13b1300c,
+  });
+  dst += write_cont(dst, 0x6d8, {
+    0xec4e4000,
+    0x0100c003,
+  });
+  dst += write_dmi(dst, &addr, s->vignetting_lut.size()*sizeof(uint32_t), 0xc24, 14); // GRR
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->vignetting_lut.size()*sizeof(uint32_t), 0xc24, 15); // GBB
+  patches.push_back(addr - (uint64_t)start);
+
+  // debayer
+  dst += write_cont(dst, 0x6f8, {
+    0x00000100,
+  });
+  dst += write_cont(dst, 0x71c, {
+    0x00008000,
+    0x08000066,
+  });
+
+  // color correction
+  dst += write_cont(dst, 0x760, s->color_correct_matrix);
+
+  // gamma
+  dst += write_cont(dst, 0x798, {
+    0x00000000,
+  });
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 26);  // G
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 28);  // B
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 30);  // R
+  patches.push_back(addr - (uint64_t)start);
+
+  // output size/scaling
+  dst += write_cont(dst, 0xa3c, {
+    0x00000003,
+    ((out_width - 1) << 16) | (s->frame_width - 1),
+    0x30036666,
+    0x00000000,
+    0x00000000,
+    s->frame_width - 1,
+    ((out_height - 1) << 16) | (s->frame_height - 1),
+    0x30036666,
+    0x00000000,
+    0x00000000,
+    s->frame_height - 1,
+  });
+  dst += write_cont(dst, 0xa68, {
+    0x00000003,
+    ((out_width / 2 - 1) << 16) | (s->frame_width - 1),
+    0x3006cccc,
+    0x00000000,
+    0x00000000,
+    s->frame_width - 1,
+    ((out_height / 2 - 1) << 16) | (s->frame_height - 1),
+    0x3006cccc,
+    0x00000000,
+    0x00000000,
+    s->frame_height - 1,
+  });
+
+  // cropping
+  dst += write_cont(dst, 0xe10, {
+    out_height - 1,
+    out_width - 1,
+  });
+  dst += write_cont(dst, 0xe30, {
+    out_height / 2 - 1,
+    out_width - 1,
+  });
+  dst += write_cont(dst, 0xe18, {
+    0x0ff00000,
+    0x00000016,
+  });
+  dst += write_cont(dst, 0xe38, {
+    0x0ff00000,
+    0x00000017,
+  });
+
+  dst += build_common_ife_bps(dst, cam, s, patches, true);
+
+  return dst - start;
+}
+
+

system/camerad/cameras/spectra.h

@@ -0,0 +1,219 @@
+#pragma once
+
+#include <sys/mman.h>
+#include <functional>
+#include <memory>
+#include <queue>
+#include <optional>
+#include <utility>
+
+#include "media/cam_req_mgr.h"
+
+#include "common/util.h"
+#include "common/swaglog.h"
+#include "system/camerad/cameras/hw.h"
+#include "system/camerad/cameras/camera_common.h"
+#include "system/camerad/sensors/sensor.h"
+
+#define MAX_IFE_BUFS 20
+
+const int MIPI_SETTLE_CNT = 33;  // Calculated by camera_freqs.py
+
+// For use with the Titan 170 ISP in the SDM845
+// https://github.com/commaai/agnos-kernel-sdm845
+
+// CSLDeviceType/CSLPacketOpcodesIFE from camx
+// cam_packet_header.op_code = (device << 24) | (opcode);
+#define CSLDeviceTypeImageSensor (0x01 << 24)
+#define CSLDeviceTypeIFE         (0x0F << 24)
+#define CSLDeviceTypeBPS         (0x10 << 24)
+#define OpcodesIFEInitialConfig  0x0
+#define OpcodesIFEUpdate         0x1
+
+std::optional<int32_t> device_acquire(int fd, int32_t session_handle, void *data, uint32_t num_resources=1);
+int device_config(int fd, int32_t session_handle, int32_t dev_handle, uint64_t packet_handle);
+int device_control(int fd, int op_code, int session_handle, int dev_handle);
+int do_cam_control(int fd, int op_code, void *handle, int size);
+void *alloc_w_mmu_hdl(int video0_fd, int len, uint32_t *handle, int align = 8, int flags = CAM_MEM_FLAG_KMD_ACCESS | CAM_MEM_FLAG_UMD_ACCESS | CAM_MEM_FLAG_CMD_BUF_TYPE,
+                      int mmu_hdl = 0, int mmu_hdl2 = 0);
+void release(int video0_fd, uint32_t handle);
+
+class MemoryManager {
+public:
+  void init(int _video0_fd) { video0_fd = _video0_fd; }
+  ~MemoryManager();
+
+  template <class T>
+  auto alloc(int len, uint32_t *handle) {
+    return std::unique_ptr<T, std::function<void(void *)>>((T*)alloc_buf(len, handle), [this](void *ptr) { this->free(ptr); });
+  }
+
+private:
+  void *alloc_buf(int len, uint32_t *handle);
+  void free(void *ptr);
+
+  std::map<void *, uint32_t> handle_lookup;
+  std::map<void *, int> size_lookup;
+  std::map<int, std::queue<void *> > cached_allocations;
+  int video0_fd;
+};
+
+class SpectraMaster {
+public:
+  void init();
+
+  unique_fd video0_fd;
+  unique_fd cam_sync_fd;
+  unique_fd isp_fd;
+  unique_fd icp_fd;
+  int device_iommu = -1;
+  int cdm_iommu = -1;
+  int icp_device_iommu = -1;
+  MemoryManager mem_mgr;
+};
+
+class SpectraBuf {
+public:
+  SpectraBuf() = default;
+
+  ~SpectraBuf() {
+    if (video_fd >= 0 && ptr) {
+      munmap(ptr, mmap_size);
+      release(video_fd, handle);
+    }
+  }
+
+  void init(SpectraMaster *m, int s, int a, bool shared_access, int mmu_hdl = 0, int mmu_hdl2 = 0, int count = 1) {
+    video_fd = m->video0_fd;
+    size = s;
+    alignment = a;
+    mmap_size = aligned_size() * count;
+
+    uint32_t flags = CAM_MEM_FLAG_HW_READ_WRITE | CAM_MEM_FLAG_KMD_ACCESS | CAM_MEM_FLAG_UMD_ACCESS | CAM_MEM_FLAG_CMD_BUF_TYPE;
+    if (shared_access) {
+      flags |= CAM_MEM_FLAG_HW_SHARED_ACCESS;
+    }
+
+    void *p = alloc_w_mmu_hdl(video_fd, mmap_size, (uint32_t*)&handle, alignment, flags, mmu_hdl, mmu_hdl2);
+    ptr = (unsigned char*)p;
+    assert(ptr != NULL);
+  };
+
+  uint32_t aligned_size() {
+    return ALIGNED_SIZE(size, alignment);
+  };
+
+  int video_fd = -1;
+  unsigned char *ptr = nullptr;
+  int size = 0, alignment = 0, handle = 0, mmap_size = 0;
+};
+
+class SpectraCamera {
+public:
+  SpectraCamera(SpectraMaster *master, const CameraConfig &config);
+  ~SpectraCamera();
+
+  void camera_open(VisionIpcServer *v, cl_device_id device_id, cl_context ctx);
+  bool handle_camera_event(const cam_req_mgr_message *event_data);
+  void camera_close();
+  void camera_map_bufs();
+  void config_bps(int idx, int request_id);
+  void config_ife(int idx, int request_id, bool init=false);
+
+  int clear_req_queue();
+  void enqueue_frame(uint64_t request_id);
+
+  int sensors_init();
+  void sensors_start();
+  void sensors_poke(int request_id);
+  void sensors_i2c(const struct i2c_random_wr_payload* dat, int len, int op_code, bool data_word);
+
+  bool openSensor();
+  void configISP();
+  void configICP();
+  void configCSIPHY();
+  void linkDevices();
+  void destroySyncObjectAt(int index);
+
+  // *** state ***
+
+  int ife_buf_depth = -1;
+  bool open = false;
+  bool enabled = true;
+  CameraConfig cc;
+  std::unique_ptr<const SensorInfo> sensor;
+
+  // YUV image size
+  uint32_t stride;
+  uint32_t y_height;
+  uint32_t uv_height;
+  uint32_t uv_offset;
+  uint32_t yuv_size;
+
+  unique_fd sensor_fd;
+  unique_fd csiphy_fd;
+
+  int32_t session_handle = -1;
+  int32_t sensor_dev_handle = -1;
+  int32_t isp_dev_handle = -1;
+  int32_t icp_dev_handle = -1;
+  int32_t csiphy_dev_handle = -1;
+
+  int32_t link_handle = -1;
+
+  SpectraBuf ife_cmd;
+  SpectraBuf ife_gamma_lut;
+  SpectraBuf ife_linearization_lut;
+  SpectraBuf ife_vignetting_lut;
+
+  SpectraBuf bps_cmd;
+  SpectraBuf bps_cdm_buffer;
+  SpectraBuf bps_cdm_program_array;
+  SpectraBuf bps_cdm_striping_bl;
+  SpectraBuf bps_iq;
+  SpectraBuf bps_striping;
+  SpectraBuf bps_linearization_lut;
+  std::vector<uint32_t> bps_lin_reg;
+  std::vector<uint32_t> bps_ccm_reg;
+
+  int buf_handle_yuv[MAX_IFE_BUFS] = {};
+  int buf_handle_raw[MAX_IFE_BUFS] = {};
+  int sync_objs_ife[MAX_IFE_BUFS] = {};
+  int sync_objs_bps[MAX_IFE_BUFS] = {};
+  uint64_t request_id_last = 0;
+  uint64_t last_requeue_ts = 0;
+  uint64_t frame_id_raw_last = 0;
+  int invalid_request_count = 0;
+  bool skip_expected = true;
+
+  CameraBuf buf;
+  SpectraMaster *m;
+
+private:
+  void clearAndRequeue(uint64_t from_request_id);
+  bool validateEvent(uint64_t request_id, uint64_t frame_id_raw);
+  bool waitForFrameReady(uint64_t request_id);
+  bool processFrame(int buf_idx, uint64_t request_id, uint64_t frame_id_raw, uint64_t timestamp);
+  static bool syncFirstFrame(int camera_id, uint64_t request_id, uint64_t raw_id, uint64_t timestamp);
+  struct SyncData {
+    uint64_t timestamp;
+    uint64_t frame_id_offset = 0;
+  };
+  inline static std::map<int, SyncData> camera_sync_data;
+  inline static bool first_frame_synced = false;
+
+  // a mode for stressing edge cases: realignment, sync failures, etc.
+  inline bool stress_test(std::string log) {
+    static double last_trigger = 0;
+    static double prob = std::stod(util::getenv("SPECTRA_ERROR_PROB", "-1"));
+    static double dt = std::stod(util::getenv("SPECTRA_ERROR_DT", "1"));
+    bool triggered = (prob > 0) && \
+                     ((static_cast<double>(rand()) / RAND_MAX) < prob) && \
+                     (millis_since_boot() - last_trigger) > dt;
+    if (triggered) {
+      last_trigger = millis_since_boot();
+      LOGE("stress test (cam %d): %s", cc.camera_num, log.c_str());
+    }
+    return triggered;
+  }
+};

system/camerad/sensors/ar0231_cl.h

@@ -0,0 +1,34 @@
+#if SENSOR_ID == 1
+
+#define VIGNETTE_PROFILE_8DT0MM
+
+#define BIT_DEPTH 12
+#define PV_MAX 4096
+#define BLACK_LVL 168
+
+float4 normalize_pv(int4 parsed, float vignette_factor) {
+  float4 pv = (convert_float4(parsed) - BLACK_LVL) / (PV_MAX - BLACK_LVL);
+  return clamp(pv*vignette_factor, 0.0, 1.0);
+}
+
+float3 color_correct(float3 rgb) {
+  float3 corrected = rgb.x * (float3)(1.82717181, -0.31231438, 0.07307673);
+  corrected += rgb.y * (float3)(-0.5743977, 1.36858544, -0.53183455);
+  corrected += rgb.z * (float3)(-0.25277411, -0.05627105, 1.45875782);
+  return corrected;
+}
+
+float3 apply_gamma(float3 rgb, int expo_time) {
+  // tone mapping params
+  const float gamma_k = 0.75;
+  const float gamma_b = 0.125;
+  const float mp = 0.01; // ideally midpoint should be adaptive
+  const float rk = 9 - 100*mp;
+
+  // poly approximation for s curve
+  return (rgb > mp) ?
+    ((rk * (rgb-mp) * (1-(gamma_k*mp+gamma_b)) * (1+1/(rk*(1-mp))) / (1+rk*(rgb-mp))) + gamma_k*mp + gamma_b) :
+    ((rk * (rgb-mp) * (gamma_k*mp+gamma_b) * (1+1/(rk*mp)) / (1-rk*(rgb-mp))) + gamma_k*mp + gamma_b);
+}
+
+#endif

system/camerad/sensors/ar0231_registers.h

@@ -0,0 +1,121 @@
+#pragma once
+
+const struct i2c_random_wr_payload start_reg_array_ar0231[] = {{0x301A, 0x91C}};
+const struct i2c_random_wr_payload stop_reg_array_ar0231[] = {{0x301A, 0x918}};
+
+const struct i2c_random_wr_payload init_array_ar0231[] = {
+  {0x301A, 0x0018}, // RESET_REGISTER
+
+  // **NOTE**: if this is changed, readout_time_ns must be updated in the Sensor config
+
+  // CLOCK Settings
+  // input clock is 19.2 / 2 * 0x37 = 528 MHz
+  // pixclk is 528 / 6 = 88 MHz
+  // full roll time is 1000/(PIXCLK/(LINE_LENGTH_PCK*FRAME_LENGTH_LINES)) = 39.99 ms
+  // img  roll time is 1000/(PIXCLK/(LINE_LENGTH_PCK*Y_OUTPUT_CONTROL))   = 22.85 ms
+  {0x302A, 0x0006}, // VT_PIX_CLK_DIV
+  {0x302C, 0x0001}, // VT_SYS_CLK_DIV
+  {0x302E, 0x0002}, // PRE_PLL_CLK_DIV
+  {0x3030, 0x0037}, // PLL_MULTIPLIER
+  {0x3036, 0x000C}, // OP_PIX_CLK_DIV
+  {0x3038, 0x0001}, // OP_SYS_CLK_DIV
+
+  // FORMAT
+  {0x3040, 0xC000}, // READ_MODE
+  {0x3004, 0x0000}, // X_ADDR_START_
+  {0x3008, 0x0787}, // X_ADDR_END_
+  {0x3002, 0x0000}, // Y_ADDR_START_
+  {0x3006, 0x04B7}, // Y_ADDR_END_
+  {0x3032, 0x0000}, // SCALING_MODE
+  {0x30A2, 0x0001}, // X_ODD_INC_
+  {0x30A6, 0x0001}, // Y_ODD_INC_
+  {0x3402, 0x0788}, // X_OUTPUT_CONTROL
+  {0x3404, 0x04B8}, // Y_OUTPUT_CONTROL
+  {0x3064, 0x1982}, // SMIA_TEST
+  {0x30BA, 0x11F2}, // DIGITAL_CTRL
+
+  // Enable external trigger and disable GPIO outputs
+  {0x30CE, 0x0120}, // SLAVE_SH_SYNC_MODE | FRAME_START_MODE
+  {0x340A, 0xE0},   // GPIO3_INPUT_DISABLE | GPIO2_INPUT_DISABLE | GPIO1_INPUT_DISABLE
+  {0x340C, 0x802},  // GPIO_HIDRV_EN | GPIO0_ISEL=2
+
+  // Readout timing
+  {0x300C, 0x0672}, // LINE_LENGTH_PCK (valid for 3-exposure HDR)
+  {0x300A, 0x0855}, // FRAME_LENGTH_LINES
+  {0x3042, 0x0000}, // EXTRA_DELAY
+
+  // Readout Settings
+  {0x31AE, 0x0204}, // SERIAL_FORMAT, 4-lane MIPI
+  {0x31AC, 0x0C0C}, // DATA_FORMAT_BITS, 12 -> 12
+  {0x3342, 0x1212}, // MIPI_F1_PDT_EDT
+  {0x3346, 0x1212}, // MIPI_F2_PDT_EDT
+  {0x334A, 0x1212}, // MIPI_F3_PDT_EDT
+  {0x334E, 0x1212}, // MIPI_F4_PDT_EDT
+  {0x3344, 0x0011}, // MIPI_F1_VDT_VC
+  {0x3348, 0x0111}, // MIPI_F2_VDT_VC
+  {0x334C, 0x0211}, // MIPI_F3_VDT_VC
+  {0x3350, 0x0311}, // MIPI_F4_VDT_VC
+  {0x31B0, 0x0053}, // FRAME_PREAMBLE
+  {0x31B2, 0x003B}, // LINE_PREAMBLE
+  {0x301A, 0x001C}, // RESET_REGISTER
+
+  // Noise Corrections
+  {0x3092, 0x0C24}, // ROW_NOISE_CONTROL
+  {0x337A, 0x0C80}, // DBLC_SCALE0
+  {0x3370, 0x03B1}, // DBLC
+  {0x3044, 0x0400}, // DARK_CONTROL
+
+  // Enable temperature sensor
+  {0x30B4, 0x0007}, // TEMPSENS0_CTRL_REG
+  {0x30B8, 0x0007}, // TEMPSENS1_CTRL_REG
+
+  // Enable dead pixel correction using
+  // the 1D line correction scheme
+  {0x31E0, 0x0003},
+
+  // HDR Settings
+  {0x3082, 0x0004}, // OPERATION_MODE_CTRL
+  {0x3238, 0x0444}, // EXPOSURE_RATIO
+
+  {0x1008, 0x0361}, // FINE_INTEGRATION_TIME_MIN
+  {0x100C, 0x0589}, // FINE_INTEGRATION_TIME2_MIN
+  {0x100E, 0x07B1}, // FINE_INTEGRATION_TIME3_MIN
+  {0x1010, 0x0139}, // FINE_INTEGRATION_TIME4_MIN
+
+  // TODO: do these have to be lower than LINE_LENGTH_PCK?
+  {0x3014, 0x08CB}, // FINE_INTEGRATION_TIME_
+  {0x321E, 0x0894}, // FINE_INTEGRATION_TIME2
+
+  {0x31D0, 0x0000}, // COMPANDING, no good in 10 bit?
+  {0x33DA, 0x0000}, // COMPANDING
+  {0x318E, 0x0200}, // PRE_HDR_GAIN_EN
+
+  // DLO Settings
+  {0x3100, 0x4000}, // DLO_CONTROL0
+  {0x3280, 0x0CCC}, // T1 G1
+  {0x3282, 0x0CCC}, // T1 R
+  {0x3284, 0x0CCC}, // T1 B
+  {0x3286, 0x0CCC}, // T1 G2
+  {0x3288, 0x0FA0}, // T2 G1
+  {0x328A, 0x0FA0}, // T2 R
+  {0x328C, 0x0FA0}, // T2 B
+  {0x328E, 0x0FA0}, // T2 G2
+
+   // Initial Gains
+  {0x3022, 0x0001}, // GROUPED_PARAMETER_HOLD_
+  {0x3366, 0xFF77}, // ANALOG_GAIN (1x)
+
+  {0x3060, 0x3333}, // ANALOG_COLOR_GAIN
+
+  {0x3362, 0x0000}, // DC GAIN
+
+  {0x305A, 0x00F8}, // red gain
+  {0x3058, 0x0122}, // blue gain
+  {0x3056, 0x009A}, // g1 gain
+  {0x305C, 0x009A}, // g2 gain
+
+  {0x3022, 0x0000}, // GROUPED_PARAMETER_HOLD_
+
+  // Initial Integration Time
+  {0x3012, 0x0005},
+};

system/camerad/sensors/os04c10_cl.h

@@ -0,0 +1,58 @@
+#if SENSOR_ID == 3
+
+#define BGGR
+#define VIGNETTE_PROFILE_4DT6MM
+
+#define BIT_DEPTH 12
+#define PV_MAX10 1023
+#define PV_MAX12 4095
+#define PV_MAX16 65536 // gamma curve is calibrated to 16bit
+#define BLACK_LVL 48
+
+float combine_dual_pvs(float lv, float sv, int expo_time) {
+  float svc = fmax(sv * expo_time, (float)(64 * (PV_MAX10 - BLACK_LVL)));
+  float svd = sv * fmin(expo_time, 8.0) / 8;
+
+  if (expo_time > 64) {
+    if (lv < PV_MAX10 - BLACK_LVL) {
+      return lv / (PV_MAX16 - BLACK_LVL);
+    } else {
+      return (svc / 64) / (PV_MAX16 - BLACK_LVL);
+    }
+  } else {
+    if (lv > 32) {
+      return (lv * 64 / fmax(expo_time, 8.0)) / (PV_MAX16 - BLACK_LVL);
+    } else {
+      return svd / (PV_MAX16 - BLACK_LVL);
+    }
+  }
+}
+
+float4 normalize_pv_hdr(int4 parsed, int4 short_parsed, float vignette_factor, int expo_time) {
+  float4 pl = convert_float4(parsed - BLACK_LVL);
+  float4 ps = convert_float4(short_parsed - BLACK_LVL);
+  float4 pv;
+  pv.s0 = combine_dual_pvs(pl.s0, ps.s0, expo_time);
+  pv.s1 = combine_dual_pvs(pl.s1, ps.s1, expo_time);
+  pv.s2 = combine_dual_pvs(pl.s2, ps.s2, expo_time);
+  pv.s3 = combine_dual_pvs(pl.s3, ps.s3, expo_time);
+  return clamp(pv*vignette_factor, 0.0, 1.0);
+}
+
+float4 normalize_pv(int4 parsed, float vignette_factor) {
+  float4 pv = (convert_float4(parsed) - BLACK_LVL) / (PV_MAX12 - BLACK_LVL);
+  return clamp(pv*vignette_factor, 0.0, 1.0);
+}
+
+float3 color_correct(float3 rgb) {
+  float3 corrected = rgb.x * (float3)(1.55361989, -0.268894615, -0.000593219);
+  corrected += rgb.y * (float3)(-0.421217301, 1.51883144, -0.69760146);
+  corrected += rgb.z * (float3)(-0.132402589, -0.249936825, 1.69819468);
+  return corrected;
+}
+
+float3 apply_gamma(float3 rgb, int expo_time) {
+  return (10 * rgb) / (1 + 9 * rgb);
+}
+
+#endif

system/camerad/sensors/os04c10_registers.h

@@ -0,0 +1,352 @@
+#pragma once
+
+const struct i2c_random_wr_payload start_reg_array_os04c10[] = {{0x100, 1}};
+const struct i2c_random_wr_payload stop_reg_array_os04c10[] = {{0x100, 0}};
+
+const struct i2c_random_wr_payload init_array_os04c10[] = {
+  // DP_2688X1520_NEWSTG_MIPI0776Mbps_30FPS_10BIT_FOURLANE
+  {0x0103, 0x01},
+
+  // PLL
+  {0x0301, 0xe4},
+  {0x0303, 0x01},
+  {0x0305, 0xb6},
+  {0x0306, 0x01},
+  {0x0307, 0x17},
+  {0x0323, 0x04},
+  {0x0324, 0x01},
+  {0x0325, 0x62},
+
+  {0x3012, 0x06},
+  {0x3013, 0x02},
+  {0x3016, 0x72},
+  {0x3021, 0x03},
+  {0x3106, 0x21},
+  {0x3107, 0xa1},
+
+  // ?
+  {0x3624, 0x00},
+  {0x3625, 0x4c},
+  {0x3660, 0x04},
+  {0x3666, 0xa5},
+  {0x3667, 0xa5},
+  {0x366a, 0x50},
+  {0x3673, 0x0d},
+  {0x3672, 0x0d},
+  {0x3671, 0x0d},
+  {0x3670, 0x0d},
+  {0x3685, 0x00},
+  {0x3694, 0x0d},
+  {0x3693, 0x0d},
+  {0x3692, 0x0d},
+  {0x3691, 0x0d},
+  {0x3696, 0x4c},
+  {0x3697, 0x4c},
+  {0x3698, 0x00},
+  {0x3699, 0x80},
+  {0x369a, 0x80},
+  {0x369b, 0x1f},
+  {0x369c, 0x1f},
+  {0x369d, 0x80},
+  {0x369e, 0x40},
+  {0x369f, 0x21},
+  {0x36a0, 0x12},
+  {0x36a1, 0xdd},
+  {0x36a2, 0x66},
+  {0x370a, 0x02},
+  {0x370e, 0x00},
+  {0x3710, 0x00},
+  {0x3713, 0x04},
+  {0x3725, 0x02},
+  {0x372a, 0x03},
+  {0x3738, 0xce},
+  {0x3748, 0x02},
+  {0x374a, 0x02},
+  {0x374c, 0x02},
+  {0x374e, 0x02},
+  {0x3756, 0x00},
+  {0x3757, 0x00},
+  {0x3767, 0x00},
+  {0x3771, 0x00},
+  {0x377b, 0x28},
+  {0x377c, 0x00},
+  {0x377d, 0x0c},
+  {0x3781, 0x03},
+  {0x3782, 0x00},
+  {0x3789, 0x14},
+  {0x3795, 0x02},
+  {0x379c, 0x00},
+  {0x379d, 0x00},
+  {0x37b8, 0x04},
+  {0x37ba, 0x03},
+  {0x37bb, 0x00},
+  {0x37bc, 0x04},
+  {0x37be, 0x26},
+  {0x37c4, 0x11},
+  {0x37c5, 0x80},
+  {0x37c6, 0x14},
+  {0x37c7, 0xa8},
+  {0x37da, 0x11},
+  {0x381f, 0x08},
+  {0x3881, 0x00},
+  {0x3888, 0x04},
+  {0x388b, 0x00},
+  {0x3c80, 0x10},
+  {0x3c86, 0x00},
+  {0x3c8c, 0x20},
+  {0x3c9f, 0x01},
+  {0x3d85, 0x1b},
+  {0x3d8c, 0x71},
+  {0x3d8d, 0xe2},
+  {0x3f00, 0x0b},
+  {0x3f06, 0x04},
+
+  // BLC
+  {0x400a, 0x01},
+  {0x400b, 0x50},
+  {0x400e, 0x08},
+  {0x4043, 0x7e},
+  {0x4045, 0x7e},
+  {0x4047, 0x7e},
+  {0x4049, 0x7e},
+  {0x4090, 0x04},
+  {0x40b0, 0x00},
+  {0x40b1, 0x00},
+  {0x40b2, 0x00},
+  {0x40b3, 0x00},
+  {0x40b4, 0x00},
+  {0x40b5, 0x00},
+  {0x40b7, 0x00},
+  {0x40b8, 0x00},
+  {0x40b9, 0x00},
+  {0x40ba, 0x01},
+
+  {0x4301, 0x00},
+  {0x4303, 0x00},
+  {0x4502, 0x04},
+  {0x4503, 0x00},
+  {0x4504, 0x06},
+  {0x4506, 0x00},
+  {0x4507, 0x47},
+  {0x4803, 0x00},
+  {0x480c, 0x32},
+  {0x480e, 0x04},
+  {0x4813, 0xe4},
+  {0x4819, 0x70},
+  {0x481f, 0x30},
+  {0x4823, 0x3f},
+  {0x4825, 0x30},
+  {0x4833, 0x10},
+  {0x484b, 0x27},
+  {0x488b, 0x00},
+  {0x4d00, 0x04},
+  {0x4d01, 0xad},
+  {0x4d02, 0xbc},
+  {0x4d03, 0xa1},
+  {0x4d04, 0x1f},
+  {0x4d05, 0x4c},
+  {0x4d0b, 0x01},
+  {0x4e00, 0x2a},
+  {0x4e0d, 0x00},
+
+  // ISP
+  {0x5001, 0x09},
+  {0x5004, 0x00},
+  {0x5080, 0x04},
+  {0x5036, 0x80},
+  {0x5180, 0x70},
+  {0x5181, 0x10},
+
+  // DPC
+  {0x520a, 0x03},
+  {0x520b, 0x06},
+  {0x520c, 0x0c},
+
+  {0x580b, 0x0f},
+  {0x580d, 0x00},
+  {0x580f, 0x00},
+  {0x5820, 0x00},
+  {0x5821, 0x00},
+
+  {0x301c, 0xf8},
+  {0x301e, 0xb4},
+  {0x301f, 0xf0},
+  {0x3022, 0x61},
+  {0x3109, 0xe7},
+  {0x3600, 0x00},
+  {0x3610, 0x65},
+  {0x3611, 0x85},
+  {0x3613, 0x3a},
+  {0x3615, 0x60},
+  {0x3621, 0xb0},
+  {0x3620, 0x0c},
+  {0x3629, 0x00},
+  {0x3661, 0x04},
+  {0x3664, 0x70},
+  {0x3665, 0x00},
+  {0x3681, 0x80},
+  {0x3682, 0x40},
+  {0x3683, 0x21},
+  {0x3684, 0x12},
+  {0x3700, 0x2a},
+  {0x3701, 0x12},
+  {0x3703, 0x28},
+  {0x3704, 0x0e},
+  {0x3706, 0x9d},
+  {0x3709, 0x4a},
+  {0x370b, 0x48},
+  {0x370c, 0x01},
+  {0x370f, 0x00},
+  {0x3714, 0x28},
+  {0x3716, 0x04},
+  {0x3719, 0x11},
+  {0x371a, 0x1e},
+  {0x3720, 0x00},
+  {0x3724, 0x13},
+  {0x373f, 0xb0},
+  {0x3741, 0x9d},
+  {0x3743, 0x9d},
+  {0x3745, 0x9d},
+  {0x3747, 0x9d},
+  {0x3749, 0x48},
+  {0x374b, 0x48},
+  {0x374d, 0x48},
+  {0x374f, 0x48},
+  {0x3755, 0x10},
+  {0x376c, 0x00},
+  {0x378d, 0x3c},
+  {0x3790, 0x01},
+  {0x3791, 0x01},
+  {0x3798, 0x40},
+  {0x379e, 0x00},
+  {0x379f, 0x04},
+  {0x37a1, 0x10},
+  {0x37a2, 0x1e},
+  {0x37a8, 0x10},
+  {0x37a9, 0x1e},
+  {0x37ac, 0xa0},
+  {0x37b9, 0x01},
+  {0x37bd, 0x01},
+  {0x37bf, 0x26},
+  {0x37c0, 0x11},
+  {0x37c2, 0x14},
+  {0x37cd, 0x19},
+  {0x37e0, 0x08},
+  {0x37e6, 0x04},
+  {0x37e5, 0x02},
+  {0x37e1, 0x0c},
+  {0x3737, 0x04},
+  {0x37d8, 0x02},
+  {0x37e2, 0x10},
+  {0x3739, 0x10},
+  {0x3662, 0x08},
+  {0x37e4, 0x20},
+  {0x37e3, 0x08},
+  {0x37d9, 0x04},
+  {0x4040, 0x00},
+  {0x4041, 0x03},
+  {0x4008, 0x01},
+  {0x4009, 0x06},
+
+  // FSIN
+  {0x3002, 0x22},
+  {0x3663, 0x22},
+  {0x368a, 0x04},
+  {0x3822, 0x44},
+  {0x3823, 0x00},
+  {0x3829, 0x03},
+  {0x3832, 0xf8},
+  {0x382c, 0x00},
+  {0x3844, 0x06},
+  {0x3843, 0x00},
+  {0x382a, 0x00},
+  {0x382b, 0x0c},
+
+  // 2704x1536 -> 2688x1520 out
+  {0x3800, 0x00}, {0x3801, 0x00},
+  {0x3802, 0x00}, {0x3803, 0x00},
+  {0x3804, 0x0a}, {0x3805, 0x8f},
+  {0x3806, 0x05}, {0x3807, 0xff},
+  {0x3808, 0x05}, {0x3809, 0x40},
+  {0x380a, 0x02}, {0x380b, 0xf8},
+  {0x3811, 0x08},
+  {0x3813, 0x08},
+  {0x3814, 0x03},
+  {0x3815, 0x01},
+  {0x3816, 0x03},
+  {0x3817, 0x01},
+
+  {0x380c, 0x0b}, {0x380d, 0xac}, // HTS
+  {0x380e, 0x06}, {0x380f, 0x9c}, // VTS
+
+  {0x3820, 0xb3},
+  {0x3821, 0x01},
+  {0x3880, 0x00},
+  {0x3882, 0x20},
+  {0x3c91, 0x0b},
+  {0x3c94, 0x45},
+  {0x3cad, 0x00},
+  {0x3cae, 0x00},
+  {0x4000, 0xf3},
+  {0x4001, 0x60},
+  {0x4003, 0x40},
+  {0x4300, 0xff},
+  {0x4302, 0x0f},
+  {0x4305, 0x83},
+  {0x4505, 0x84},
+  {0x4809, 0x0e},
+  {0x480a, 0x04},
+  {0x4837, 0x15},
+  {0x4c00, 0x08},
+  {0x4c01, 0x08},
+  {0x4c04, 0x00},
+  {0x4c05, 0x00},
+  {0x5000, 0xf9},
+  // {0x0100, 0x01},
+  // {0x320d, 0x00},
+  // {0x3208, 0xa0},
+
+  // initialize exposure
+  {0x3503, 0x88},
+
+  // long
+  {0x3500, 0x00}, {0x3501, 0x00}, {0x3502, 0x10},
+  {0x3508, 0x00}, {0x3509, 0x80},
+  {0x350a, 0x04}, {0x350b, 0x00},
+
+  // short
+  {0x3510, 0x00}, {0x3511, 0x00}, {0x3512, 0x40},
+  {0x350c, 0x00}, {0x350d, 0x80},
+  {0x350e, 0x04}, {0x350f, 0x00},
+
+  // wb
+  // b
+  {0x5100, 0x06}, {0x5101, 0x7e},
+  {0x5140, 0x06}, {0x5141, 0x7e},
+  // g
+  {0x5102, 0x04}, {0x5103, 0x00},
+  {0x5142, 0x04}, {0x5143, 0x00},
+  // r
+  {0x5104, 0x08}, {0x5105, 0xd6},
+  {0x5144, 0x08}, {0x5145, 0xd6},
+};
+
+const struct i2c_random_wr_payload ife_downscale_override_array_os04c10[] = {
+  // OS04C10_AA_00_02_17_wAO_2688x1524_MIPI728Mbps_Linear12bit_20FPS_4Lane_MCLK24MHz
+  {0x3c8c, 0x40},
+  {0x3714, 0x24},
+  {0x37c2, 0x04},
+  {0x3662, 0x10},
+  {0x37d9, 0x08},
+  {0x4041, 0x07},
+  {0x4008, 0x02},
+  {0x4009, 0x0d},
+  {0x3808, 0x0a}, {0x3809, 0x80},
+  {0x380a, 0x05}, {0x380b, 0xf0},
+  {0x3814, 0x01},
+  {0x3816, 0x01},
+  {0x380c, 0x08}, {0x380d, 0x5c}, // HTS
+  {0x380e, 0x09}, {0x380f, 0x38}, // VTS
+  {0x3820, 0xb0},
+  {0x3821, 0x00},
+};

system/camerad/sensors/ox03c10_cl.h

@@ -0,0 +1,47 @@
+#if SENSOR_ID == 2
+
+#define VIGNETTE_PROFILE_8DT0MM
+
+#define BIT_DEPTH 12
+#define BLACK_LVL 64
+
+float ox_lut_func(int x) {
+  if (x < 512) {
+    return x * 5.94873e-8;
+  } else if (512 <= x && x < 768) {
+    return 3.0458e-05 + (x-512) * 1.19913e-7;
+  } else if (768 <= x && x < 1536) {
+    return 6.1154e-05 + (x-768) * 2.38493e-7;
+  } else if (1536 <= x && x < 1792) {
+    return 0.0002448 + (x-1536) * 9.56930e-7;
+  } else if (1792 <= x && x < 2048) {
+    return 0.00048977 + (x-1792) * 1.91441e-6;
+  } else if (2048 <= x && x < 2304) {
+    return 0.00097984 + (x-2048) * 3.82937e-6;
+  } else if (2304 <= x && x < 2560) {
+    return 0.0019601 + (x-2304) * 7.659055e-6;
+  } else if (2560 <= x && x < 2816) {
+    return 0.0039207 + (x-2560) * 1.525e-5;
+  } else {
+    return 0.0078421 + (exp((x-2816)/273.0) - 1) * 0.0092421;
+  }
+}
+
+float4 normalize_pv(int4 parsed, float vignette_factor) {
+  // PWL
+  float4 pv = {ox_lut_func(parsed.s0), ox_lut_func(parsed.s1), ox_lut_func(parsed.s2), ox_lut_func(parsed.s3)};
+  return clamp(pv*vignette_factor*256.0, 0.0, 1.0);
+}
+
+float3 color_correct(float3 rgb) {
+  float3 corrected = rgb.x * (float3)(1.5664815, -0.29808738, -0.03973474);
+  corrected += rgb.y * (float3)(-0.48672447, 1.41914433, -0.40295248);
+  corrected += rgb.z * (float3)(-0.07975703, -0.12105695, 1.44268722);
+  return corrected;
+}
+
+float3 apply_gamma(float3 rgb, int expo_time) {
+  return -0.507089*exp(-12.54124638*rgb) + 0.9655*powr(rgb, 0.5) - 0.472597*rgb + 0.507089;
+}
+
+#endif

system/camerad/sensors/ox03c10_registers.h

@@ -0,0 +1,751 @@
+#pragma once
+
+const struct i2c_random_wr_payload start_reg_array_ox03c10[] = {{0x100, 1}};
+const struct i2c_random_wr_payload stop_reg_array_ox03c10[] = {{0x100, 0}};
+
+const struct i2c_random_wr_payload init_array_ox03c10[] = {
+  {0x103, 1},
+  {0x107, 1},
+
+  // X3C_1920x1280_60fps_HDR4_LFR_PWL12_mipi1200
+
+  // TPM
+  {0x4d5a, 0x1a}, {0x4d09, 0xff}, {0x4d09, 0xdf},
+
+  /*)
+  // group 4
+  {0x3208, 0x04},
+  {0x4620, 0x04},
+  {0x3208, 0x14},
+
+  // group 5
+  {0x3208, 0x05},
+  {0x4620, 0x04},
+  {0x3208, 0x15},
+
+  // group 2
+  {0x3208, 0x02},
+  {0x3507, 0x00},
+  {0x3208, 0x12},
+
+  // delay launch group 2
+  {0x3208, 0xa2},*/
+
+  // **NOTE**: if this is changed, readout_time_ns must be updated in the Sensor config
+  // PLL setup
+  {0x0301, 0xc8}, // pll1_divs, pll1_predivp, pll1_divpix
+  {0x0303, 0x01}, // pll1_prediv
+  {0x0304, 0x01}, {0x0305, 0x2c}, // pll1_loopdiv = 300
+  {0x0306, 0x04}, // pll1_divmipi = 4
+  {0x0307, 0x01}, // pll1_divm = 1
+  {0x0316, 0x00},
+  {0x0317, 0x00},
+  {0x0318, 0x00},
+  {0x0323, 0x05}, // pll2_prediv
+  {0x0324, 0x01}, {0x0325, 0x2c}, // pll2_divp = 300
+
+  // SCLK/PCLK
+  {0x0400, 0xe0}, {0x0401, 0x80},
+  {0x0403, 0xde}, {0x0404, 0x34},
+  {0x0405, 0x3b}, {0x0406, 0xde},
+  {0x0407, 0x08},
+  {0x0408, 0xe0}, {0x0409, 0x7f},
+  {0x040a, 0xde}, {0x040b, 0x34},
+  {0x040c, 0x47}, {0x040d, 0xd8},
+  {0x040e, 0x08},
+
+  // xchk
+  {0x2803, 0xfe}, {0x280b, 0x00}, {0x280c, 0x79},
+
+  // SC ctrl
+  {0x3001, 0x03}, // io_pad_oen
+  {0x3002, 0xfc}, // io_pad_oen
+  {0x3005, 0x80}, // io_pad_out
+  {0x3007, 0x01}, // io_pad_sel
+  {0x3008, 0x80}, // io_pad_sel
+
+  // FSIN (frame sync) with external pulses
+  {0x3009, 0x2},
+  {0x3015, 0x2},
+  {0x383E, 0x80},
+  {0x3881, 0x4},
+  {0x3882, 0x8}, {0x3883, 0x0D},
+  {0x3836, 0x1F}, {0x3837, 0x40},
+
+  // causes issues on some devices
+  //{0x3822, 0x33},  // wait for pulse before first frame
+
+  {0x3892, 0x44},
+  {0x3823, 0x41},
+
+  {0x3012, 0x41}, // SC_PHY_CTRL = 4 lane MIPI
+  {0x3020, 0x05}, // SC_CTRL_20
+
+  // this is not in the datasheet, listed as RSVD
+  // but the camera doesn't work without it
+  {0x3700, 0x28}, {0x3701, 0x15}, {0x3702, 0x19}, {0x3703, 0x23},
+  {0x3704, 0x0a}, {0x3705, 0x00}, {0x3706, 0x3e}, {0x3707, 0x0d},
+  {0x3708, 0x50}, {0x3709, 0x5a}, {0x370a, 0x00}, {0x370b, 0x96},
+  {0x3711, 0x11}, {0x3712, 0x13}, {0x3717, 0x02}, {0x3718, 0x73},
+  {0x372c, 0x40}, {0x3733, 0x01}, {0x3738, 0x36}, {0x3739, 0x36},
+  {0x373a, 0x25}, {0x373b, 0x25}, {0x373f, 0x21}, {0x3740, 0x21},
+  {0x3741, 0x21}, {0x3742, 0x21}, {0x3747, 0x28}, {0x3748, 0x28},
+  {0x3749, 0x19}, {0x3755, 0x1a}, {0x3756, 0x0a}, {0x3757, 0x1c},
+  {0x3765, 0x19}, {0x3766, 0x05}, {0x3767, 0x05}, {0x3768, 0x13},
+  {0x376c, 0x07}, {0x3778, 0x20}, {0x377c, 0xc8}, {0x3781, 0x02},
+  {0x3783, 0x02}, {0x379c, 0x58}, {0x379e, 0x00}, {0x379f, 0x00},
+  {0x37a0, 0x00}, {0x37bc, 0x22}, {0x37c0, 0x01}, {0x37c4, 0x3e},
+  {0x37c5, 0x3e}, {0x37c6, 0x2a}, {0x37c7, 0x28}, {0x37c8, 0x02},
+  {0x37c9, 0x12}, {0x37cb, 0x29}, {0x37cd, 0x29}, {0x37d2, 0x00},
+  {0x37d3, 0x73}, {0x37d6, 0x00}, {0x37d7, 0x6b}, {0x37dc, 0x00},
+  {0x37df, 0x54}, {0x37e2, 0x00}, {0x37e3, 0x00}, {0x37f8, 0x00},
+  {0x37f9, 0x01}, {0x37fa, 0x00}, {0x37fb, 0x19},
+
+  // also RSVD
+  {0x3c03, 0x01}, {0x3c04, 0x01}, {0x3c06, 0x21}, {0x3c08, 0x01},
+  {0x3c09, 0x01}, {0x3c0a, 0x01}, {0x3c0b, 0x21}, {0x3c13, 0x21},
+  {0x3c14, 0x82}, {0x3c16, 0x13}, {0x3c21, 0x00}, {0x3c22, 0xf3},
+  {0x3c37, 0x12}, {0x3c38, 0x31}, {0x3c3c, 0x00}, {0x3c3d, 0x03},
+  {0x3c44, 0x16}, {0x3c5c, 0x8a}, {0x3c5f, 0x03}, {0x3c61, 0x80},
+  {0x3c6f, 0x2b}, {0x3c70, 0x5f}, {0x3c71, 0x2c}, {0x3c72, 0x2c},
+  {0x3c73, 0x2c}, {0x3c76, 0x12},
+
+  // PEC checks
+  {0x3182, 0x12},
+
+  {0x320e, 0x00}, {0x320f, 0x00}, // RSVD
+  {0x3211, 0x61},
+  {0x3215, 0xcd},
+  {0x3219, 0x08},
+
+  {0x3506, 0x20}, {0x3507, 0x00}, // hcg fine exposure
+  {0x350a, 0x01}, {0x350b, 0x00}, {0x350c, 0x00}, // hcg digital gain
+
+  {0x3586, 0x40}, {0x3587, 0x00}, // lcg fine exposure
+  {0x358a, 0x01}, {0x358b, 0x00}, {0x358c, 0x00}, // lcg digital gain
+
+  {0x3546, 0x20}, {0x3547, 0x00}, // spd fine exposure
+  {0x354a, 0x01}, {0x354b, 0x00}, {0x354c, 0x00}, // spd digital gain
+
+  {0x35c6, 0xb0}, {0x35c7, 0x00}, // vs fine exposure
+  {0x35ca, 0x01}, {0x35cb, 0x00}, {0x35cc, 0x00}, // vs digital gain
+
+  // also RSVD
+  {0x3600, 0x8f}, {0x3605, 0x16}, {0x3609, 0xf0}, {0x360a, 0x01},
+  {0x360e, 0x1d}, {0x360f, 0x10}, {0x3610, 0x70}, {0x3611, 0x3a},
+  {0x3612, 0x28}, {0x361a, 0x29}, {0x361b, 0x6c}, {0x361c, 0x0b},
+  {0x361d, 0x00}, {0x361e, 0xfc}, {0x362a, 0x00}, {0x364d, 0x0f},
+  {0x364e, 0x18}, {0x364f, 0x12}, {0x3653, 0x1c}, {0x3654, 0x00},
+  {0x3655, 0x1f}, {0x3656, 0x1f}, {0x3657, 0x0c}, {0x3658, 0x0a},
+  {0x3659, 0x14}, {0x365a, 0x18}, {0x365b, 0x14}, {0x365c, 0x10},
+  {0x365e, 0x12}, {0x3674, 0x08}, {0x3677, 0x3a}, {0x3678, 0x3a},
+  {0x3679, 0x19},
+
+  // Y_ADDR_START = 4
+  {0x3802, 0x00}, {0x3803, 0x04},
+  // Y_ADDR_END = 0x50b
+  {0x3806, 0x05}, {0x3807, 0x0b},
+
+  // X_OUTPUT_SIZE = 0x780 = 1920 (changed to 1928)
+  {0x3808, 0x07}, {0x3809, 0x88},
+
+  // Y_OUTPUT_SIZE = 0x500 = 1280 (changed to 1208)
+  {0x380a, 0x04}, {0x380b, 0xb8},
+
+  // horizontal timing 0x447
+  {0x380c, 0x04}, {0x380d, 0x47},
+
+  // rows per frame (was 0x2ae)
+  // 0x8ae = 53.65 ms
+  {0x380e, 0x08}, {0x380f, 0x15},
+  // this should be triggered by FSIN, not free running
+
+  {0x3810, 0x00}, {0x3811, 0x08}, // x cutoff
+  {0x3812, 0x00}, {0x3813, 0x04}, // y cutoff
+  {0x3816, 0x01},
+  {0x3817, 0x01},
+  {0x381c, 0x18},
+  {0x381e, 0x01},
+  {0x381f, 0x01},
+
+  // don't mirror, just flip
+  {0x3820, 0x04},
+
+  {0x3821, 0x19},
+  {0x3832, 0xF0},
+  {0x3834, 0xF0},
+  {0x384c, 0x02},
+  {0x384d, 0x0d},
+  {0x3850, 0x00},
+  {0x3851, 0x42},
+  {0x3852, 0x00},
+  {0x3853, 0x40},
+  {0x3858, 0x04},
+  {0x388c, 0x02},
+  {0x388d, 0x2b},
+
+  // APC
+  {0x3b40, 0x05}, {0x3b41, 0x40}, {0x3b42, 0x00}, {0x3b43, 0x90},
+  {0x3b44, 0x00}, {0x3b45, 0x20}, {0x3b46, 0x00}, {0x3b47, 0x20},
+  {0x3b48, 0x19}, {0x3b49, 0x12}, {0x3b4a, 0x16}, {0x3b4b, 0x2e},
+  {0x3b4c, 0x00}, {0x3b4d, 0x00},
+  {0x3b86, 0x00}, {0x3b87, 0x34}, {0x3b88, 0x00}, {0x3b89, 0x08},
+  {0x3b8a, 0x05}, {0x3b8b, 0x00}, {0x3b8c, 0x07}, {0x3b8d, 0x80},
+  {0x3b8e, 0x00}, {0x3b8f, 0x00}, {0x3b92, 0x05}, {0x3b93, 0x00},
+  {0x3b94, 0x07}, {0x3b95, 0x80}, {0x3b9e, 0x09},
+
+  // OTP
+  {0x3d82, 0x73},
+  {0x3d85, 0x05},
+  {0x3d8a, 0x03},
+  {0x3d8b, 0xff},
+  {0x3d99, 0x00},
+  {0x3d9a, 0x9f},
+  {0x3d9b, 0x00},
+  {0x3d9c, 0xa0},
+  {0x3da4, 0x00},
+  {0x3da7, 0x50},
+
+  // DTR
+  {0x420e, 0x6b},
+  {0x420f, 0x6e},
+  {0x4210, 0x06},
+  {0x4211, 0xc1},
+  {0x421e, 0x02},
+  {0x421f, 0x45},
+  {0x4220, 0xe1},
+  {0x4221, 0x01},
+  {0x4301, 0xff},
+  {0x4307, 0x03},
+  {0x4308, 0x13},
+  {0x430a, 0x13},
+  {0x430d, 0x93},
+  {0x430f, 0x57},
+  {0x4310, 0x95},
+  {0x4311, 0x16},
+  {0x4316, 0x00},
+
+  {0x4317, 0x38}, // both embedded rows are enabled
+
+  {0x4319, 0x03}, // spd dcg
+  {0x431a, 0x00}, // 8 bit mipi
+  {0x431b, 0x00},
+  {0x431d, 0x2a},
+  {0x431e, 0x11},
+
+  {0x431f, 0x20}, // enable PWL (pwl0_en), 12 bits
+  //{0x431f, 0x00}, // disable PWL
+
+  {0x4320, 0x19},
+  {0x4323, 0x80},
+  {0x4324, 0x00},
+  {0x4503, 0x4e},
+  {0x4505, 0x00},
+  {0x4509, 0x00},
+  {0x450a, 0x00},
+  {0x4580, 0xf8},
+  {0x4583, 0x07},
+  {0x4584, 0x6a},
+  {0x4585, 0x08},
+  {0x4586, 0x05},
+  {0x4587, 0x04},
+  {0x4588, 0x73},
+  {0x4589, 0x05},
+  {0x458a, 0x1f},
+  {0x458b, 0x02},
+  {0x458c, 0xdc},
+  {0x458d, 0x03},
+  {0x458e, 0x02},
+  {0x4597, 0x07},
+  {0x4598, 0x40},
+  {0x4599, 0x0e},
+  {0x459a, 0x0e},
+  {0x459b, 0xfb},
+  {0x459c, 0xf3},
+  {0x4602, 0x00},
+  {0x4603, 0x13},
+  {0x4604, 0x00},
+  {0x4609, 0x0a},
+  {0x460a, 0x30},
+  {0x4610, 0x00},
+  {0x4611, 0x70},
+  {0x4612, 0x01},
+  {0x4613, 0x00},
+  {0x4614, 0x00},
+  {0x4615, 0x70},
+  {0x4616, 0x01},
+  {0x4617, 0x00},
+
+  {0x4800, 0x04}, // invert output PCLK
+  {0x480a, 0x22},
+  {0x4813, 0xe4},
+
+  // mipi
+  {0x4814, 0x2a},
+  {0x4837, 0x0d},
+  {0x484b, 0x47},
+  {0x484f, 0x00},
+  {0x4887, 0x51},
+  {0x4d00, 0x4a},
+  {0x4d01, 0x18},
+  {0x4d05, 0xff},
+  {0x4d06, 0x88},
+  {0x4d08, 0x63},
+  {0x4d09, 0xdf},
+  {0x4d15, 0x7d},
+  {0x4d1a, 0x20},
+  {0x4d30, 0x0a},
+  {0x4d31, 0x00},
+  {0x4d34, 0x7d},
+  {0x4d3c, 0x7d},
+  {0x4f00, 0x00},
+  {0x4f01, 0x00},
+  {0x4f02, 0x00},
+  {0x4f03, 0x20},
+  {0x4f04, 0xe0},
+  {0x6a00, 0x00},
+  {0x6a01, 0x20},
+  {0x6a02, 0x00},
+  {0x6a03, 0x20},
+  {0x6a04, 0x02},
+  {0x6a05, 0x80},
+  {0x6a06, 0x01},
+  {0x6a07, 0xe0},
+  {0x6a08, 0xcf},
+  {0x6a09, 0x01},
+  {0x6a0a, 0x40},
+  {0x6a20, 0x00},
+  {0x6a21, 0x02},
+  {0x6a22, 0x00},
+  {0x6a23, 0x00},
+  {0x6a24, 0x00},
+  {0x6a25, 0x00},
+  {0x6a26, 0x00},
+  {0x6a27, 0x00},
+  {0x6a28, 0x00},
+
+  // isp
+  {0x5000, 0x8f},
+  {0x5001, 0x75},
+  {0x5002, 0x7f}, // PWL0
+  //{0x5002, 0x3f}, // PWL disable
+  {0x5003, 0x7a},
+
+  {0x5004, 0x3e},
+  {0x5005, 0x1e},
+  {0x5006, 0x1e},
+  {0x5007, 0x1e},
+
+  {0x5008, 0x00},
+  {0x500c, 0x00},
+  {0x502c, 0x00},
+  {0x502e, 0x00},
+  {0x502f, 0x00},
+  {0x504b, 0x00},
+  {0x5053, 0x00},
+  {0x505b, 0x00},
+  {0x5063, 0x00},
+  {0x5070, 0x00},
+  {0x5074, 0x04},
+  {0x507a, 0x04},
+  {0x507b, 0x09},
+  {0x5500, 0x02},
+  {0x5700, 0x02},
+  {0x5900, 0x02},
+  {0x6007, 0x04},
+  {0x6008, 0x05},
+  {0x6009, 0x02},
+  {0x600b, 0x08},
+  {0x600c, 0x07},
+  {0x600d, 0x88},
+  {0x6016, 0x00},
+  {0x6027, 0x04},
+  {0x6028, 0x05},
+  {0x6029, 0x02},
+  {0x602b, 0x08},
+  {0x602c, 0x07},
+  {0x602d, 0x88},
+  {0x6047, 0x04},
+  {0x6048, 0x05},
+  {0x6049, 0x02},
+  {0x604b, 0x08},
+  {0x604c, 0x07},
+  {0x604d, 0x88},
+  {0x6067, 0x04},
+  {0x6068, 0x05},
+  {0x6069, 0x02},
+  {0x606b, 0x08},
+  {0x606c, 0x07},
+  {0x606d, 0x88},
+  {0x6087, 0x04},
+  {0x6088, 0x05},
+  {0x6089, 0x02},
+  {0x608b, 0x08},
+  {0x608c, 0x07},
+  {0x608d, 0x88},
+
+  // 12-bit PWL0
+  {0x5e00, 0x00},
+
+  // m_ndX_exp[0:32]
+  // 9*2+0xa*3+0xb*2+0xc*2+0xd*2+0xe*2+0xf*2+0x10*2+0x11*2+0x12*4+0x13*3+0x14*3+0x15*3+0x16 = 518
+  {0x5e01, 0x09},
+  {0x5e02, 0x09},
+  {0x5e03, 0x0a},
+  {0x5e04, 0x0a},
+  {0x5e05, 0x0a},
+  {0x5e06, 0x0b},
+  {0x5e07, 0x0b},
+  {0x5e08, 0x0c},
+  {0x5e09, 0x0c},
+  {0x5e0a, 0x0d},
+  {0x5e0b, 0x0d},
+  {0x5e0c, 0x0e},
+  {0x5e0d, 0x0e},
+  {0x5e0e, 0x0f},
+  {0x5e0f, 0x0f},
+  {0x5e10, 0x10},
+  {0x5e11, 0x10},
+  {0x5e12, 0x11},
+  {0x5e13, 0x11},
+  {0x5e14, 0x12},
+  {0x5e15, 0x12},
+  {0x5e16, 0x12},
+  {0x5e17, 0x12},
+  {0x5e18, 0x13},
+  {0x5e19, 0x13},
+  {0x5e1a, 0x13},
+  {0x5e1b, 0x14},
+  {0x5e1c, 0x14},
+  {0x5e1d, 0x14},
+  {0x5e1e, 0x15},
+  {0x5e1f, 0x15},
+  {0x5e20, 0x15},
+  {0x5e21, 0x16},
+
+  // m_ndY_val[0:32]
+  // 0x200+0xff+0x100*3+0x80*12+0x40*16 = 4095
+  {0x5e22, 0x00}, {0x5e23, 0x02}, {0x5e24, 0x00},
+  {0x5e25, 0x00}, {0x5e26, 0x00}, {0x5e27, 0xff},
+  {0x5e28, 0x00}, {0x5e29, 0x01}, {0x5e2a, 0x00},
+  {0x5e2b, 0x00}, {0x5e2c, 0x01}, {0x5e2d, 0x00},
+  {0x5e2e, 0x00}, {0x5e2f, 0x01}, {0x5e30, 0x00},
+  {0x5e31, 0x00}, {0x5e32, 0x00}, {0x5e33, 0x80},
+  {0x5e34, 0x00}, {0x5e35, 0x00}, {0x5e36, 0x80},
+  {0x5e37, 0x00}, {0x5e38, 0x00}, {0x5e39, 0x80},
+  {0x5e3a, 0x00}, {0x5e3b, 0x00}, {0x5e3c, 0x80},
+  {0x5e3d, 0x00}, {0x5e3e, 0x00}, {0x5e3f, 0x80},
+  {0x5e40, 0x00}, {0x5e41, 0x00}, {0x5e42, 0x80},
+  {0x5e43, 0x00}, {0x5e44, 0x00}, {0x5e45, 0x80},
+  {0x5e46, 0x00}, {0x5e47, 0x00}, {0x5e48, 0x80},
+  {0x5e49, 0x00}, {0x5e4a, 0x00}, {0x5e4b, 0x80},
+  {0x5e4c, 0x00}, {0x5e4d, 0x00}, {0x5e4e, 0x80},
+  {0x5e4f, 0x00}, {0x5e50, 0x00}, {0x5e51, 0x80},
+  {0x5e52, 0x00}, {0x5e53, 0x00}, {0x5e54, 0x80},
+  {0x5e55, 0x00}, {0x5e56, 0x00}, {0x5e57, 0x40},
+  {0x5e58, 0x00}, {0x5e59, 0x00}, {0x5e5a, 0x40},
+  {0x5e5b, 0x00}, {0x5e5c, 0x00}, {0x5e5d, 0x40},
+  {0x5e5e, 0x00}, {0x5e5f, 0x00}, {0x5e60, 0x40},
+  {0x5e61, 0x00}, {0x5e62, 0x00}, {0x5e63, 0x40},
+  {0x5e64, 0x00}, {0x5e65, 0x00}, {0x5e66, 0x40},
+  {0x5e67, 0x00}, {0x5e68, 0x00}, {0x5e69, 0x40},
+  {0x5e6a, 0x00}, {0x5e6b, 0x00}, {0x5e6c, 0x40},
+  {0x5e6d, 0x00}, {0x5e6e, 0x00}, {0x5e6f, 0x40},
+  {0x5e70, 0x00}, {0x5e71, 0x00}, {0x5e72, 0x40},
+  {0x5e73, 0x00}, {0x5e74, 0x00}, {0x5e75, 0x40},
+  {0x5e76, 0x00}, {0x5e77, 0x00}, {0x5e78, 0x40},
+  {0x5e79, 0x00}, {0x5e7a, 0x00}, {0x5e7b, 0x40},
+  {0x5e7c, 0x00}, {0x5e7d, 0x00}, {0x5e7e, 0x40},
+  {0x5e7f, 0x00}, {0x5e80, 0x00}, {0x5e81, 0x40},
+  {0x5e82, 0x00}, {0x5e83, 0x00}, {0x5e84, 0x40},
+
+  // disable PWL
+  /*{0x5e01, 0x18}, {0x5e02, 0x00}, {0x5e03, 0x00}, {0x5e04, 0x00},
+  {0x5e05, 0x00}, {0x5e06, 0x00}, {0x5e07, 0x00}, {0x5e08, 0x00},
+  {0x5e09, 0x00}, {0x5e0a, 0x00}, {0x5e0b, 0x00}, {0x5e0c, 0x00},
+  {0x5e0d, 0x00}, {0x5e0e, 0x00}, {0x5e0f, 0x00}, {0x5e10, 0x00},
+  {0x5e11, 0x00}, {0x5e12, 0x00}, {0x5e13, 0x00}, {0x5e14, 0x00},
+  {0x5e15, 0x00}, {0x5e16, 0x00}, {0x5e17, 0x00}, {0x5e18, 0x00},
+  {0x5e19, 0x00}, {0x5e1a, 0x00}, {0x5e1b, 0x00}, {0x5e1c, 0x00},
+  {0x5e1d, 0x00}, {0x5e1e, 0x00}, {0x5e1f, 0x00}, {0x5e20, 0x00},
+  {0x5e21, 0x00},
+
+  {0x5e22, 0x00}, {0x5e23, 0x0f}, {0x5e24, 0xFF},*/
+
+  {0x4001, 0x2b},  // BLC_CTRL_1
+  {0x4008, 0x02}, {0x4009, 0x03},
+  {0x4018, 0x12},
+  {0x4022, 0x40},
+  {0x4023, 0x20},
+
+  // all black level targets are 0x40
+  {0x4026, 0x00}, {0x4027, 0x40},
+  {0x4028, 0x00}, {0x4029, 0x40},
+  {0x402a, 0x00}, {0x402b, 0x40},
+  {0x402c, 0x00}, {0x402d, 0x40},
+
+  {0x407e, 0xcc},
+  {0x407f, 0x18},
+  {0x4080, 0xff},
+  {0x4081, 0xff},
+  {0x4082, 0x01},
+  {0x4083, 0x53},
+  {0x4084, 0x01},
+  {0x4085, 0x2b},
+  {0x4086, 0x00},
+  {0x4087, 0xb3},
+
+  {0x4640, 0x40},
+  {0x4641, 0x11},
+  {0x4642, 0x0e},
+  {0x4643, 0xee},
+  {0x4646, 0x0f},
+  {0x4648, 0x00},
+  {0x4649, 0x03},
+
+  {0x4f00, 0x00},
+  {0x4f01, 0x00},
+  {0x4f02, 0x80},
+  {0x4f03, 0x2c},
+  {0x4f04, 0xf8},
+
+  {0x4d09, 0xff},
+  {0x4d09, 0xdf},
+
+  {0x5003, 0x7a},
+  {0x5b80, 0x08},
+  {0x5c00, 0x08},
+  {0x5c80, 0x00},
+  {0x5bbe, 0x12},
+  {0x5c3e, 0x12},
+  {0x5cbe, 0x12},
+  {0x5b8a, 0x80},
+  {0x5b8b, 0x80},
+  {0x5b8c, 0x80},
+  {0x5b8d, 0x80},
+  {0x5b8e, 0x60},
+  {0x5b8f, 0x80},
+  {0x5b90, 0x80},
+  {0x5b91, 0x80},
+  {0x5b92, 0x80},
+  {0x5b93, 0x20},
+  {0x5b94, 0x80},
+  {0x5b95, 0x80},
+  {0x5b96, 0x80},
+  {0x5b97, 0x20},
+  {0x5b98, 0x00},
+  {0x5b99, 0x80},
+  {0x5b9a, 0x40},
+  {0x5b9b, 0x20},
+  {0x5b9c, 0x00},
+  {0x5b9d, 0x00},
+  {0x5b9e, 0x80},
+  {0x5b9f, 0x00},
+  {0x5ba0, 0x00},
+  {0x5ba1, 0x00},
+  {0x5ba2, 0x00},
+  {0x5ba3, 0x00},
+  {0x5ba4, 0x00},
+  {0x5ba5, 0x00},
+  {0x5ba6, 0x00},
+  {0x5ba7, 0x00},
+  {0x5ba8, 0x02},
+  {0x5ba9, 0x00},
+  {0x5baa, 0x02},
+  {0x5bab, 0x76},
+  {0x5bac, 0x03},
+  {0x5bad, 0x08},
+  {0x5bae, 0x00},
+  {0x5baf, 0x80},
+  {0x5bb0, 0x00},
+  {0x5bb1, 0xc0},
+  {0x5bb2, 0x01},
+  {0x5bb3, 0x00},
+
+  // m_nNormCombineWeight
+  {0x5c0a, 0x80}, {0x5c0b, 0x80}, {0x5c0c, 0x80}, {0x5c0d, 0x80}, {0x5c0e, 0x60},
+  {0x5c0f, 0x80}, {0x5c10, 0x80}, {0x5c11, 0x80}, {0x5c12, 0x60}, {0x5c13, 0x20},
+  {0x5c14, 0x80}, {0x5c15, 0x80}, {0x5c16, 0x80}, {0x5c17, 0x20}, {0x5c18, 0x00},
+  {0x5c19, 0x80}, {0x5c1a, 0x40}, {0x5c1b, 0x20}, {0x5c1c, 0x00}, {0x5c1d, 0x00},
+  {0x5c1e, 0x80}, {0x5c1f, 0x00}, {0x5c20, 0x00}, {0x5c21, 0x00}, {0x5c22, 0x00},
+  {0x5c23, 0x00}, {0x5c24, 0x00}, {0x5c25, 0x00}, {0x5c26, 0x00}, {0x5c27, 0x00},
+
+  // m_nCombinThreL
+  {0x5c28, 0x02}, {0x5c29, 0x00},
+  {0x5c2a, 0x02}, {0x5c2b, 0x76},
+  {0x5c2c, 0x03}, {0x5c2d, 0x08},
+
+  // m_nCombinThreS
+  {0x5c2e, 0x00}, {0x5c2f, 0x80},
+  {0x5c30, 0x00}, {0x5c31, 0xc0},
+  {0x5c32, 0x01}, {0x5c33, 0x00},
+
+  // m_nNormCombineWeight
+  {0x5c8a, 0x80}, {0x5c8b, 0x80}, {0x5c8c, 0x80}, {0x5c8d, 0x80}, {0x5c8e, 0x80},
+  {0x5c8f, 0x80}, {0x5c90, 0x80}, {0x5c91, 0x80}, {0x5c92, 0x80}, {0x5c93, 0x60},
+  {0x5c94, 0x80}, {0x5c95, 0x80}, {0x5c96, 0x80}, {0x5c97, 0x60}, {0x5c98, 0x40},
+  {0x5c99, 0x80}, {0x5c9a, 0x80}, {0x5c9b, 0x80}, {0x5c9c, 0x40}, {0x5c9d, 0x00},
+  {0x5c9e, 0x80}, {0x5c9f, 0x80}, {0x5ca0, 0x80}, {0x5ca1, 0x20}, {0x5ca2, 0x00},
+  {0x5ca3, 0x80}, {0x5ca4, 0x80}, {0x5ca5, 0x00}, {0x5ca6, 0x00}, {0x5ca7, 0x00},
+
+  {0x5ca8, 0x01}, {0x5ca9, 0x00},
+  {0x5caa, 0x02}, {0x5cab, 0x00},
+  {0x5cac, 0x03}, {0x5cad, 0x08},
+
+  {0x5cae, 0x01}, {0x5caf, 0x00},
+  {0x5cb0, 0x02}, {0x5cb1, 0x00},
+  {0x5cb2, 0x03}, {0x5cb3, 0x08},
+
+  // combine ISP
+  {0x5be7, 0x80},
+  {0x5bc9, 0x80},
+  {0x5bca, 0x80},
+  {0x5bcb, 0x80},
+  {0x5bcc, 0x80},
+  {0x5bcd, 0x80},
+  {0x5bce, 0x80},
+  {0x5bcf, 0x80},
+  {0x5bd0, 0x80},
+  {0x5bd1, 0x80},
+  {0x5bd2, 0x20},
+  {0x5bd3, 0x80},
+  {0x5bd4, 0x40},
+  {0x5bd5, 0x20},
+  {0x5bd6, 0x00},
+  {0x5bd7, 0x00},
+  {0x5bd8, 0x00},
+  {0x5bd9, 0x00},
+  {0x5bda, 0x00},
+  {0x5bdb, 0x00},
+  {0x5bdc, 0x00},
+  {0x5bdd, 0x00},
+  {0x5bde, 0x00},
+  {0x5bdf, 0x00},
+  {0x5be0, 0x00},
+  {0x5be1, 0x00},
+  {0x5be2, 0x00},
+  {0x5be3, 0x00},
+  {0x5be4, 0x00},
+  {0x5be5, 0x00},
+  {0x5be6, 0x00},
+
+  // m_nSPDCombineWeight
+  {0x5c49, 0x80}, {0x5c4a, 0x80}, {0x5c4b, 0x80}, {0x5c4c, 0x80}, {0x5c4d, 0x40},
+  {0x5c4e, 0x80}, {0x5c4f, 0x80}, {0x5c50, 0x80}, {0x5c51, 0x60}, {0x5c52, 0x20},
+  {0x5c53, 0x80}, {0x5c54, 0x80}, {0x5c55, 0x80}, {0x5c56, 0x20}, {0x5c57, 0x00},
+  {0x5c58, 0x80}, {0x5c59, 0x40}, {0x5c5a, 0x20}, {0x5c5b, 0x00}, {0x5c5c, 0x00},
+  {0x5c5d, 0x80}, {0x5c5e, 0x00}, {0x5c5f, 0x00}, {0x5c60, 0x00}, {0x5c61, 0x00},
+  {0x5c62, 0x00}, {0x5c63, 0x00}, {0x5c64, 0x00}, {0x5c65, 0x00}, {0x5c66, 0x00},
+
+  // m_nSPDCombineWeight
+  {0x5cc9, 0x80}, {0x5cca, 0x80}, {0x5ccb, 0x80}, {0x5ccc, 0x80}, {0x5ccd, 0x80},
+  {0x5cce, 0x80}, {0x5ccf, 0x80}, {0x5cd0, 0x80}, {0x5cd1, 0x80}, {0x5cd2, 0x60},
+  {0x5cd3, 0x80}, {0x5cd4, 0x80}, {0x5cd5, 0x80}, {0x5cd6, 0x60}, {0x5cd7, 0x40},
+  {0x5cd8, 0x80}, {0x5cd9, 0x80}, {0x5cda, 0x80}, {0x5cdb, 0x40}, {0x5cdc, 0x20},
+  {0x5cdd, 0x80}, {0x5cde, 0x80}, {0x5cdf, 0x80}, {0x5ce0, 0x20}, {0x5ce1, 0x00},
+  {0x5ce2, 0x80}, {0x5ce3, 0x80}, {0x5ce4, 0x80}, {0x5ce5, 0x00}, {0x5ce6, 0x00},
+
+  {0x5d74, 0x01},
+  {0x5d75, 0x00},
+
+  {0x5d1f, 0x81},
+  {0x5d11, 0x00},
+  {0x5d12, 0x10},
+  {0x5d13, 0x10},
+  {0x5d15, 0x05},
+  {0x5d16, 0x05},
+  {0x5d17, 0x05},
+  {0x5d08, 0x03},
+  {0x5d09, 0xb6},
+  {0x5d0a, 0x03},
+  {0x5d0b, 0xb6},
+  {0x5d18, 0x03},
+  {0x5d19, 0xb6},
+  {0x5d62, 0x01},
+  {0x5d40, 0x02},
+  {0x5d41, 0x01},
+  {0x5d63, 0x1f},
+  {0x5d64, 0x00},
+  {0x5d65, 0x80},
+  {0x5d56, 0x00},
+  {0x5d57, 0x20},
+  {0x5d58, 0x00},
+  {0x5d59, 0x20},
+  {0x5d5a, 0x00},
+  {0x5d5b, 0x0c},
+  {0x5d5c, 0x02},
+  {0x5d5d, 0x40},
+  {0x5d5e, 0x02},
+  {0x5d5f, 0x40},
+  {0x5d60, 0x03},
+  {0x5d61, 0x40},
+  {0x5d4a, 0x02},
+  {0x5d4b, 0x40},
+  {0x5d4c, 0x02},
+  {0x5d4d, 0x40},
+  {0x5d4e, 0x02},
+  {0x5d4f, 0x40},
+  {0x5d50, 0x18},
+  {0x5d51, 0x80},
+  {0x5d52, 0x18},
+  {0x5d53, 0x80},
+  {0x5d54, 0x18},
+  {0x5d55, 0x80},
+  {0x5d46, 0x20},
+  {0x5d47, 0x00},
+  {0x5d48, 0x22},
+  {0x5d49, 0x00},
+  {0x5d42, 0x20},
+  {0x5d43, 0x00},
+  {0x5d44, 0x22},
+  {0x5d45, 0x00},
+
+  {0x5004, 0x1e},
+  {0x4221, 0x03},  // this is changed from 1 -> 3
+
+  // DCG exposure coarse
+  // {0x3501, 0x01}, {0x3502, 0xc8},
+  // SPD exposure coarse
+  // {0x3541, 0x01}, {0x3542, 0xc8},
+  // VS exposure coarse
+  // {0x35c1, 0x00}, {0x35c2, 0x01},
+
+  // crc reference
+  {0x420e, 0x66}, {0x420f, 0x5d}, {0x4210, 0xa8}, {0x4211, 0x55},
+  // crc stat check
+  {0x507a, 0x5f}, {0x507b, 0x46},
+
+  // watchdog control
+  {0x4f00, 0x00}, {0x4f01, 0x01}, {0x4f02, 0x80}, {0x4f04, 0x2c},
+
+  // color balance gains
+  // blue
+  {0x5280, 0x06}, {0x5281, 0xCB}, // hcg
+  {0x5480, 0x06}, {0x5481, 0xCB}, // lcg
+  {0x5680, 0x06}, {0x5681, 0xCB}, // spd
+  {0x5880, 0x06}, {0x5881, 0xCB}, // vs
+
+  // green(blue)
+  {0x5282, 0x04}, {0x5283, 0x00},
+  {0x5482, 0x04}, {0x5483, 0x00},
+  {0x5682, 0x04}, {0x5683, 0x00},
+  {0x5882, 0x04}, {0x5883, 0x00},
+
+  // green(red)
+  {0x5284, 0x04}, {0x5285, 0x00},
+  {0x5484, 0x04}, {0x5485, 0x00},
+  {0x5684, 0x04}, {0x5685, 0x00},
+  {0x5884, 0x04}, {0x5885, 0x00},
+
+  // red
+  {0x5286, 0x08}, {0x5287, 0xDE},
+  {0x5486, 0x08}, {0x5487, 0xDE},
+  {0x5686, 0x08}, {0x5687, 0xDE},
+  {0x5886, 0x08}, {0x5887, 0xDE},
+
+  // fixed gains
+  {0x3588, 0x01}, {0x3589, 0x00},
+  {0x35c8, 0x01}, {0x35c9, 0x00},
+  {0x3548, 0x0F}, {0x3549, 0x00},
+  {0x35c1, 0x00},
+};

system/camerad/sensors/sensor.h

@@ -0,0 +1,117 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <map>
+#include <utility>
+#include <vector>
+
+#include "media/cam_isp.h"
+#include "media/cam_sensor.h"
+
+#include "cereal/gen/cpp/log.capnp.h"
+#include "system/camerad/sensors/ar0231_registers.h"
+#include "system/camerad/sensors/ox03c10_registers.h"
+#include "system/camerad/sensors/os04c10_registers.h"
+
+#define ANALOG_GAIN_MAX_CNT 55
+
+class SensorInfo {
+public:
+  SensorInfo() = default;
+  virtual std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const { return {}; }
+  virtual float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const {return 0; }
+  virtual int getSlaveAddress(int port) const { assert(0); }
+
+  cereal::FrameData::ImageSensor image_sensor = cereal::FrameData::ImageSensor::UNKNOWN;
+  float pixel_size_mm;
+  uint32_t frame_width, frame_height;
+  uint32_t frame_stride;
+  uint32_t frame_offset = 0;
+  uint32_t extra_height = 0;
+  int out_scale = 1;
+  int registers_offset = -1;
+  int stats_offset = -1;
+  int hdr_offset = -1;
+
+  int exposure_time_min;
+  int exposure_time_max;
+
+  float dc_gain_factor;
+  int dc_gain_min_weight;
+  int dc_gain_max_weight;
+  float dc_gain_on_grey;
+  float dc_gain_off_grey;
+
+  float ev_scale = 1.0;
+  float sensor_analog_gains[ANALOG_GAIN_MAX_CNT];
+  int analog_gain_min_idx;
+  int analog_gain_max_idx;
+  int analog_gain_rec_idx;
+  int analog_gain_cost_delta;
+  float analog_gain_cost_low;
+  float analog_gain_cost_high;
+  float target_grey_factor;
+  float min_ev;
+  float max_ev;
+
+  bool data_word;
+  uint32_t probe_reg_addr;
+  uint32_t probe_expected_data;
+  std::vector<i2c_random_wr_payload> start_reg_array;
+  std::vector<i2c_random_wr_payload> init_reg_array;
+
+  uint32_t bits_per_pixel;
+  uint32_t bayer_pattern;
+  uint32_t mipi_format;
+  uint32_t mclk_frequency;
+  uint32_t frame_data_type;
+
+  uint32_t readout_time_ns;  // used to recover EOF from SOF
+
+  // ISP image processing params
+  uint32_t black_level;
+  std::vector<uint32_t> color_correct_matrix;  // 3x3
+  std::vector<uint32_t> gamma_lut_rgb;         // gamma LUTs are length 64 * sizeof(uint32_t); same for r/g/b here
+  void prepare_gamma_lut() {
+    for (int i = 0; i < 64; i++) {
+      gamma_lut_rgb[i] |= ((uint32_t)(gamma_lut_rgb[i+1] - gamma_lut_rgb[i]) << 10);
+    }
+    gamma_lut_rgb.pop_back();
+  }
+  std::vector<uint32_t> linearization_lut;     // length 36
+  std::vector<uint32_t> linearization_pts;     // length 4
+  std::vector<uint32_t> vignetting_lut;        // length 221
+
+  const int num() const {
+    return static_cast<int>(image_sensor);
+  };
+};
+
+class AR0231 : public SensorInfo {
+public:
+  AR0231();
+  std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const override;
+  float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const override;
+  int getSlaveAddress(int port) const override;
+
+private:
+  mutable std::map<uint16_t, std::pair<int, int>> ar0231_register_lut;
+};
+
+class OX03C10 : public SensorInfo {
+public:
+  OX03C10();
+  std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const override;
+  float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const override;
+  int getSlaveAddress(int port) const override;
+};
+
+class OS04C10 : public SensorInfo {
+public:
+  OS04C10();
+  void ife_downscale_configure();
+  std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const override;
+  float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const override;
+  int getSlaveAddress(int port) const override;
+};

system/camerad/snapshot/snapshot.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import subprocess
+import time
+
+import numpy as np
+from PIL import Image
+
+import cereal.messaging as messaging
+from msgq.visionipc import VisionIpcClient, VisionStreamType
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_MDL
+from openpilot.system.hardware import PC
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from openpilot.system.manager.process_config import managed_processes
+
+
+VISION_STREAMS = {
+  "roadCameraState": VisionStreamType.VISION_STREAM_ROAD,
+  "driverCameraState": VisionStreamType.VISION_STREAM_DRIVER,
+  "wideRoadCameraState": VisionStreamType.VISION_STREAM_WIDE_ROAD,
+}
+
+
+def jpeg_write(fn, dat):
+  img = Image.fromarray(dat)
+  img.save(fn, "JPEG")
+
+
+def yuv_to_rgb(y, u, v):
+  ul = np.repeat(np.repeat(u, 2).reshape(u.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+  vl = np.repeat(np.repeat(v, 2).reshape(v.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+
+  yuv = np.dstack((y, ul, vl)).astype(np.int16)
+  yuv[:, :, 1:] -= 128
+
+  m = np.array([
+    [1.00000,  1.00000, 1.00000],
+    [0.00000, -0.39465, 2.03211],
+    [1.13983, -0.58060, 0.00000],
+  ])
+  rgb = np.dot(yuv, m).clip(0, 255)
+  return rgb.astype(np.uint8)
+
+
+def extract_image(buf):
+  y = np.array(buf.data[:buf.uv_offset], dtype=np.uint8).reshape((-1, buf.stride))[:buf.height, :buf.width]
+  u = np.array(buf.data[buf.uv_offset::2], dtype=np.uint8).reshape((-1, buf.stride//2))[:buf.height//2, :buf.width//2]
+  v = np.array(buf.data[buf.uv_offset+1::2], dtype=np.uint8).reshape((-1, buf.stride//2))[:buf.height//2, :buf.width//2]
+
+  return yuv_to_rgb(y, u, v)
+
+
+def get_snapshots(frame="roadCameraState", front_frame="driverCameraState"):
+  sockets = [s for s in (frame, front_frame) if s is not None]
+  sm = messaging.SubMaster(sockets)
+  vipc_clients = {s: VisionIpcClient("camerad", VISION_STREAMS[s], True) for s in sockets}
+
+  # wait 4 sec from camerad startup for focus and exposure
+  while sm[sockets[0]].frameId < int(4. / DT_MDL):
+    sm.update()
+
+  for client in vipc_clients.values():
+    client.connect(True)
+
+  # grab images
+  rear, front = None, None
+  if frame is not None:
+    c = vipc_clients[frame]
+    rear = extract_image(c.recv())
+  if front_frame is not None:
+    c = vipc_clients[front_frame]
+    front = extract_image(c.recv())
+  return rear, front
+
+
+def snapshot():
+  params = Params()
+
+  if (not params.get_bool("IsOffroad")) or params.get_bool("IsTakingSnapshot"):
+    print("Already taking snapshot")
+    return None, None
+
+  front_camera_allowed = params.get_bool("RecordFront")
+  params.put_bool("IsTakingSnapshot", True)
+  set_offroad_alert("Offroad_IsTakingSnapshot", True)
+  time.sleep(2.0)  # Give hardwared time to read the param, or if just started give camerad time to start
+
+  # Check if camerad is already started
+  try:
+    subprocess.check_call(["pgrep", "camerad"])
+    print("Camerad already running")
+    params.put_bool("IsTakingSnapshot", False)
+    params.remove("Offroad_IsTakingSnapshot")
+    return None, None
+  except subprocess.CalledProcessError:
+    pass
+
+  try:
+    # Allow testing on replay on PC
+    if not PC:
+      managed_processes['camerad'].start()
+
+    frame = "wideRoadCameraState"
+    front_frame = "driverCameraState" if front_camera_allowed else None
+    rear, front = get_snapshots(frame, front_frame)
+  finally:
+    managed_processes['camerad'].stop()
+    params.put_bool("IsTakingSnapshot", False)
+    set_offroad_alert("Offroad_IsTakingSnapshot", False)
+
+  if not front_camera_allowed:
+    front = None
+
+  return rear, front
+
+
+if __name__ == "__main__":
+  pic, fpic = snapshot()
+  if pic is not None:
+    print(pic.shape)
+    jpeg_write("/tmp/back.jpg", pic)
+    if fpic is not None:
+      jpeg_write("/tmp/front.jpg", fpic)
+  else:
+    print("Error taking snapshot")

system/camerad/test/check_skips.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# type: ignore
+import cereal.messaging as messaging
+
+all_sockets = ['roadCameraState', 'driverCameraState', 'wideRoadCameraState']
+prev_id = [None,None,None]
+this_id = [None,None,None]
+dt = [None,None,None]
+num_skipped = [0,0,0]
+
+if __name__ == "__main__":
+  sm = messaging.SubMaster(all_sockets)
+  while True:
+    sm.update()
+
+    for i in range(len(all_sockets)):
+      if not sm.updated[all_sockets[i]]:
+        continue
+      this_id[i] = sm[all_sockets[i]].frameId
+      if prev_id[i] is None:
+        prev_id[i] = this_id[i]
+        continue
+      dt[i] = this_id[i] - prev_id[i]
+      if dt[i] != 1:
+        num_skipped[i] += dt[i] - 1
+        print(all_sockets[i] ,dt[i] - 1, num_skipped[i])
+      prev_id[i] = this_id[i]

system/camerad/test/debug.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+#echo 4294967295 | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+
+# no CCI and UTIL, very spammy
+echo 0xfffdbfff | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+#echo 0 | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+
+sudo dmesg -C
+scons -u -j8 --minimal .
+export DEBUG_FRAMES=1
+export DISABLE_ROAD=1 DISABLE_WIDE_ROAD=1
+#export DISABLE_DRIVER=1
+export LOGPRINT=debug
+./camerad

system/camerad/test/get_thumbnails_for_segment.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+import argparse
+import os
+from tqdm import tqdm
+
+from openpilot.tools.lib.logreader import LogReader
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("route", help="The route name")
+  args = parser.parse_args()
+
+  out_path = os.path.join("jpegs", f"{args.route.replace('|', '_').replace('/', '_')}")
+  os.makedirs(out_path, exist_ok=True)
+
+  lr = LogReader(args.route)
+
+  for msg in tqdm(lr):
+    if msg.which() == 'thumbnail':
+      with open(os.path.join(out_path, f"{msg.thumbnail.frameId}.jpg"), 'wb') as f:
+        f.write(msg.thumbnail.thumbnail)
+    elif msg.which() == 'navThumbnail':
+      with open(os.path.join(out_path, f"nav_{msg.navThumbnail.frameId}.jpg"), 'wb') as f:
+        f.write(msg.navThumbnail.thumbnail)

system/camerad/test/icp_debug.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+cd /sys/kernel/debug/tracing
+echo "" > trace
+echo 1 > tracing_on
+#echo Y > /sys/kernel/debug/camera_icp/a5_debug_q
+echo 0x1 > /sys/kernel/debug/camera_icp/a5_debug_type
+echo 1 > /sys/kernel/debug/tracing/events/camera/enable
+echo 0xffffffff > /sys/kernel/debug/camera_icp/a5_debug_lvl
+echo 1 > /sys/kernel/debug/tracing/events/camera/cam_icp_fw_dbg/enable
+
+cat /sys/kernel/debug/tracing/trace_pipe

system/camerad/test/intercept.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+DISABLE_ROAD=1 DISABLE_WIDE_ROAD=1 DEBUG_FRAMES=1 LOGPRINT=debug LD_PRELOAD=/data/tici_test_scripts/isp/interceptor/tmpioctl.so ./camerad

system/camerad/test/stress_restart.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+cd ..
+while :; do
+  ./camerad &
+  pid="$!"
+  sleep 2
+  kill -2 $pid
+  wait $pid
+done

system/camerad/test/test_camerad.py

@@ -0,0 +1,98 @@
+import os
+import time
+import pytest
+import numpy as np
+
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.tools.lib.log_time_series import msgs_to_time_series
+
+TEST_TIMESPAN = 10
+CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
+
+
+def run_and_log(procs, services, duration):
+  logs = []
+
+  try:
+    for p in procs:
+      managed_processes[p].start()
+    socks = [messaging.sub_sock(s, conflate=False, timeout=100) for s in services]
+
+    start_time = time.monotonic()
+    while time.monotonic() - start_time < duration:
+      for s in socks:
+        logs.extend(messaging.drain_sock(s))
+    for p in procs:
+      assert managed_processes[p].proc.is_alive()
+  finally:
+    for p in procs:
+      managed_processes[p].stop()
+
+  return logs
+
+@pytest.fixture(scope="module")
+def logs():
+  logs = run_and_log(["camerad", ], CAMERAS, TEST_TIMESPAN)
+  ts = msgs_to_time_series(logs)
+
+  for cam in CAMERAS:
+    expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN
+    cnt = len(ts[cam]['t'])
+    assert expected_frames*0.8 < cnt < expected_frames*1.2, f"unexpected frame count {cam}: {expected_frames=}, got {cnt}"
+
+    dts = np.abs(np.diff([ts[cam]['timestampSof']/1e6]) - 1000/SERVICE_LIST[cam].frequency)
+    assert (dts < 1.0).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
+  return ts
+
+@pytest.mark.tici
+class TestCamerad:
+  def test_frame_skips(self, logs):
+    for c in CAMERAS:
+      assert set(np.diff(logs[c]['frameId'])) == {1, }, f"{c} has frame skips"
+
+  def test_frame_sync(self, logs):
+    n = range(len(logs['roadCameraState']['t'][:-10]))
+
+    frame_ids = {i: [logs[cam]['frameId'][i] for cam in CAMERAS] for i in n}
+    assert all(len(set(v)) == 1 for v in frame_ids.values()), "frame IDs not aligned"
+
+    frame_times = {i: [logs[cam]['timestampSof'][i] for cam in CAMERAS] for i in n}
+    diffs = {i: (max(ts) - min(ts))/1e6 for i, ts in frame_times.items()}
+
+    laggy_frames = {k: v for k, v in diffs.items() if v > 1.1}
+    assert len(laggy_frames) == 0, f"Frames not synced properly: {laggy_frames=}"
+
+  def test_sanity_checks(self, logs):
+    self._sanity_checks(logs)
+
+  def _sanity_checks(self, ts):
+    for c in CAMERAS:
+      assert c in ts
+      assert len(ts[c]['t']) > 20
+
+      # not a valid request id
+      assert 0 not in ts[c]['requestId']
+
+      # should monotonically increase
+      assert np.all(np.diff(ts[c]['frameId']) >= 1)
+      assert np.all(np.diff(ts[c]['requestId']) >= 1)
+
+      # EOF > SOF
+      assert np.all((ts[c]['timestampEof'] - ts[c]['timestampSof']) > 0)
+
+      # logMonoTime > SOF
+      assert np.all((ts[c]['t'] - ts[c]['timestampSof']/1e9) > 1e-7)
+      assert np.all((ts[c]['t'] - ts[c]['timestampEof']/1e9) > 1e-7)
+
+  def test_stress_test(self):
+    os.environ['SPECTRA_ERROR_PROB'] = '0.008'
+    logs = run_and_log(["camerad", ], CAMERAS, 10)
+    ts = msgs_to_time_series(logs)
+
+    # we should see some jumps from introduced errors
+    assert np.max([ np.max(np.diff(ts[c]['frameId'])) for c in CAMERAS ]) > 1
+    assert np.max([ np.max(np.diff(ts[c]['requestId'])) for c in CAMERAS ]) > 1
+
+    self._sanity_checks(ts)

system/camerad/test/test_exposure.py

@@ -0,0 +1,51 @@
+import time
+import numpy as np
+import pytest
+
+from openpilot.selfdrive.test.helpers import with_processes
+from openpilot.system.camerad.snapshot.snapshot import get_snapshots
+
+TEST_TIME = 45
+REPEAT = 5
+
+@pytest.mark.tici
+class TestCamerad:
+  @classmethod
+  def setup_class(cls):
+    pass
+
+  def _numpy_rgb2gray(self, im):
+    ret = np.clip(im[:,:,2] * 0.114 + im[:,:,1] * 0.587 + im[:,:,0] * 0.299, 0, 255).astype(np.uint8)
+    return ret
+
+  def _is_exposure_okay(self, i, med_mean=None):
+    if med_mean is None:
+      med_mean = np.array([[0.2,0.4],[0.2,0.6]])
+    h, w = i.shape[:2]
+    i = i[h//10:9*h//10,w//10:9*w//10]
+    med_ex, mean_ex = med_mean
+    i = self._numpy_rgb2gray(i)
+    i_median = np.median(i) / 255.
+    i_mean = np.mean(i) / 255.
+    print([i_median, i_mean])
+    return med_ex[0] < i_median < med_ex[1] and mean_ex[0] < i_mean < mean_ex[1]
+
+  @with_processes(['camerad'])
+  def test_camera_operation(self):
+    passed = 0
+    start = time.time()
+    while time.time() - start < TEST_TIME and passed < REPEAT:
+      rpic, dpic = get_snapshots(frame="roadCameraState", front_frame="driverCameraState")
+      wpic, _ = get_snapshots(frame="wideRoadCameraState")
+
+      res = self._is_exposure_okay(rpic)
+      res = res and self._is_exposure_okay(dpic)
+      res = res and self._is_exposure_okay(wpic)
+
+      if passed > 0 and not res:
+        passed = -passed # fails test if any failure after first sus
+        break
+
+      passed += int(res)
+      time.sleep(2)
+    assert passed >= REPEAT

system/hardware/__init__.py

@@ -0,0 +1,16 @@
+import os
+from typing import cast
+
+from openpilot.system.hardware.base import HardwareBase
+from openpilot.system.hardware.tici.hardware import Tici
+from openpilot.system.hardware.pc.hardware import Pc
+
+TICI = os.path.isfile('/TICI')
+AGNOS = os.path.isfile('/AGNOS')
+PC = not TICI
+
+
+if TICI:
+  HARDWARE = cast(HardwareBase, Tici())
+else:
+  HARDWARE = cast(HardwareBase, Pc())

system/hardware/base.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <cstdlib>
+#include <fstream>
+#include <map>
+#include <string>
+
+#include "cereal/gen/cpp/log.capnp.h"
+
+// no-op base hw class
+class HardwareNone {
+public:
+  static constexpr float MAX_VOLUME = 0.7;
+  static constexpr float MIN_VOLUME = 0.2;
+
+  static std::string get_os_version() { return ""; }
+  static std::string get_name() { return ""; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
+  static int get_voltage() { return 0; }
+  static int get_current() { return 0; }
+
+  static std::string get_serial() { return "cccccc"; }
+
+  static std::map<std::string, std::string> get_init_logs() {
+    return {};
+  }
+
+  static void reboot() {}
+  static void poweroff() {}
+  static void set_brightness(int percent) {}
+  static void set_ir_power(int percentage) {}
+  static void set_display_power(bool on) {}
+
+  static bool get_ssh_enabled() { return false; }
+  static void set_ssh_enabled(bool enabled) {}
+
+  static void config_cpu_rendering(bool offscreen);
+
+  static bool PC() { return false; }
+  static bool TICI() { return false; }
+  static bool AGNOS() { return false; }
+};

system/hardware/base.py

@@ -0,0 +1,230 @@
+import os
+from abc import abstractmethod, ABC
+from dataclasses import dataclass, fields
+
+from cereal import log
+
+NetworkType = log.DeviceState.NetworkType
+
+class LPAError(RuntimeError):
+  pass
+
+class LPAProfileNotFoundError(LPAError):
+  pass
+
+@dataclass
+class Profile:
+  iccid: str
+  nickname: str
+  enabled: bool
+  provider: str
+
+@dataclass
+class ThermalZone:
+  # a zone from /sys/class/thermal/thermal_zone*
+  name: str             # a.k.a type
+  scale: float = 1000.  # scale to get degrees in C
+  zone_number = -1
+
+  def read(self) -> float:
+    if self.zone_number < 0:
+      for n in os.listdir("/sys/devices/virtual/thermal"):
+        if not n.startswith("thermal_zone"):
+          continue
+        with open(os.path.join("/sys/devices/virtual/thermal", n, "type")) as f:
+          if f.read().strip() == self.name:
+            self.zone_number = int(n.removeprefix("thermal_zone"))
+            break
+
+    try:
+      with open(f"/sys/devices/virtual/thermal/thermal_zone{self.zone_number}/temp") as f:
+        return int(f.read()) / self.scale
+    except FileNotFoundError:
+      return 0
+
+@dataclass
+class ThermalConfig:
+  cpu: list[ThermalZone] | None = None
+  gpu: list[ThermalZone] | None = None
+  dsp: ThermalZone | None = None
+  pmic: list[ThermalZone] | None = None
+  memory: ThermalZone | None = None
+  intake: ThermalZone | None = None
+  exhaust: ThermalZone | None = None
+  case: ThermalZone | None = None
+
+  def get_msg(self):
+    ret = {}
+    for f in fields(ThermalConfig):
+      v = getattr(self, f.name)
+      if v is not None:
+        if isinstance(v, list):
+          ret[f.name + "TempC"] = [x.read() for x in v]
+        else:
+          ret[f.name + "TempC"] = v.read()
+    return ret
+
+class LPABase(ABC):
+  @abstractmethod
+  def list_profiles(self) -> list[Profile]:
+    pass
+
+  @abstractmethod
+  def get_active_profile(self) -> Profile | None:
+    pass
+
+  @abstractmethod
+  def delete_profile(self, iccid: str) -> None:
+    pass
+
+  @abstractmethod
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    pass
+
+  @abstractmethod
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    pass
+
+  @abstractmethod
+  def switch_profile(self, iccid: str) -> None:
+    pass
+
+class HardwareBase(ABC):
+  @staticmethod
+  def get_cmdline() -> dict[str, str]:
+    with open('/proc/cmdline') as f:
+      cmdline = f.read()
+    return {kv[0]: kv[1] for kv in [s.split('=') for s in cmdline.split(' ')] if len(kv) == 2}
+
+  @staticmethod
+  def read_param_file(path, parser, default=0):
+    try:
+      with open(path) as f:
+        return parser(f.read())
+    except Exception:
+      return default
+
+  def booted(self) -> bool:
+    return True
+
+  @abstractmethod
+  def reboot(self, reason=None):
+    pass
+
+  @abstractmethod
+  def uninstall(self):
+    pass
+
+  @abstractmethod
+  def get_os_version(self):
+    pass
+
+  @abstractmethod
+  def get_device_type(self):
+    pass
+
+  @abstractmethod
+  def get_imei(self, slot) -> str:
+    pass
+
+  @abstractmethod
+  def get_serial(self):
+    pass
+
+  @abstractmethod
+  def get_network_info(self):
+    pass
+
+  @abstractmethod
+  def get_network_type(self):
+    pass
+
+  @abstractmethod
+  def get_sim_info(self):
+    pass
+
+  @abstractmethod
+  def get_sim_lpa(self) -> LPABase:
+    pass
+
+  @abstractmethod
+  def get_network_strength(self, network_type):
+    pass
+
+  def get_network_metered(self, network_type) -> bool:
+    return network_type not in (NetworkType.none, NetworkType.wifi, NetworkType.ethernet)
+
+  @staticmethod
+  def set_bandwidth_limit(upload_speed_kbps: int, download_speed_kbps: int) -> None:
+    pass
+
+  @abstractmethod
+  def get_current_power_draw(self):
+    pass
+
+  @abstractmethod
+  def get_som_power_draw(self):
+    pass
+
+  @abstractmethod
+  def shutdown(self):
+    pass
+
+  def get_thermal_config(self):
+    return ThermalConfig()
+
+  def set_display_power(self, on: bool):
+    pass
+
+  @abstractmethod
+  def set_screen_brightness(self, percentage):
+    pass
+
+  @abstractmethod
+  def get_screen_brightness(self):
+    pass
+
+  @abstractmethod
+  def set_power_save(self, powersave_enabled):
+    pass
+
+  @abstractmethod
+  def get_gpu_usage_percent(self):
+    pass
+
+  def get_modem_version(self):
+    return None
+
+  @abstractmethod
+  def get_modem_temperatures(self):
+    pass
+
+  @abstractmethod
+  def get_nvme_temperatures(self):
+    pass
+
+  @abstractmethod
+  def initialize_hardware(self):
+    pass
+
+  def configure_modem(self):
+    pass
+
+  def reboot_modem(self):
+    pass
+
+  @abstractmethod
+  def get_networks(self):
+    pass
+
+  def has_internal_panda(self) -> bool:
+    return False
+
+  def reset_internal_panda(self):
+    pass
+
+  def recover_internal_panda(self):
+    pass
+
+  def get_modem_data_usage(self):
+    return -1, -1

system/hardware/esim.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import argparse
+import time
+from openpilot.system.hardware import HARDWARE
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(prog='esim.py', description='manage eSIM profiles on your comma device', epilog='comma.ai')
+  parser.add_argument('--backend', choices=['qmi', 'at'], default='qmi', help='use the specified backend, defaults to qmi')
+  parser.add_argument('--switch', metavar='iccid', help='switch to profile')
+  parser.add_argument('--delete', metavar='iccid', help='delete profile (warning: this cannot be undone)')
+  parser.add_argument('--download', nargs=2, metavar=('qr', 'name'), help='download a profile using QR code (format: LPA:1$rsp.truphone.com$QRF-SPEEDTEST)')
+  parser.add_argument('--nickname', nargs=2, metavar=('iccid', 'name'), help='update the nickname for a profile')
+  args = parser.parse_args()
+
+  mutated = False
+  lpa = HARDWARE.get_sim_lpa()
+  if args.switch:
+    lpa.switch_profile(args.switch)
+    mutated = True
+  elif args.delete:
+    confirm = input('are you sure you want to delete this profile? (y/N) ')
+    if confirm == 'y':
+      lpa.delete_profile(args.delete)
+      mutated = True
+    else:
+      print('cancelled')
+      exit(0)
+  elif args.download:
+    lpa.download_profile(args.download[0], args.download[1])
+  elif args.nickname:
+    lpa.nickname_profile(args.nickname[0], args.nickname[1])
+  else:
+    parser.print_help()
+
+  if mutated:
+    HARDWARE.reboot_modem()
+    # eUICC needs a small delay post-reboot before querying profiles
+    time.sleep(.5)
+
+  profiles = lpa.list_profiles()
+  print(f'\n{len(profiles)} profile{"s" if len(profiles) > 1 else ""}:')
+  for p in profiles:
+    print(f'- {p.iccid} (nickname: {p.nickname or "<none provided>"}) (provider: {p.provider}) - {"enabled" if p.enabled else "disabled"}')

system/hardware/fan_controller.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import numpy as np
+from abc import ABC, abstractmethod
+
+from openpilot.common.realtime import DT_HW
+from openpilot.common.swaglog import cloudlog
+from openpilot.common.pid import PIDController
+
+class BaseFanController(ABC):
+  @abstractmethod
+  def update(self, cur_temp: float, ignition: bool) -> int:
+    pass
+
+
+class TiciFanController(BaseFanController):
+  def __init__(self) -> None:
+    super().__init__()
+    cloudlog.info("Setting up TICI fan handler")
+
+    self.last_ignition = False
+    self.controller = PIDController(k_p=0, k_i=4e-3, k_f=1, rate=(1 / DT_HW))
+
+  def update(self, cur_temp: float, ignition: bool) -> int:
+    self.controller.neg_limit = -(100 if ignition else 30)
+    self.controller.pos_limit = -(30 if ignition else 0)
+
+    if ignition != self.last_ignition:
+      self.controller.reset()
+
+    error = 75 - cur_temp
+    fan_pwr_out = -int(self.controller.update(
+                      error=error,
+                      feedforward=np.interp(cur_temp, [60.0, 100.0], [0, -100])
+                    ))
+
+    self.last_ignition = ignition
+    return fan_pwr_out
+

system/hardware/hardwared.py

@@ -0,0 +1,488 @@
+#!/usr/bin/env python3
+import fcntl
+import os
+import json
+import queue
+import struct
+import threading
+import time
+from collections import OrderedDict, namedtuple
+from pathlib import Path
+
+import psutil
+
+import cereal.messaging as messaging
+from cereal import log
+from cereal.services import SERVICE_LIST
+from openpilot.common.dict_helpers import strip_deprecated_keys
+from openpilot.common.filter_simple import FirstOrderFilter
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_HW
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from openpilot.system.hardware import HARDWARE, TICI, AGNOS
+from openpilot.system.loggerd.config import get_available_percent
+from openpilot.system.statsd import statlog
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware.power_monitoring import PowerMonitoring
+from openpilot.system.hardware.fan_controller import TiciFanController
+from openpilot.system.version import terms_version, training_version
+
+ThermalStatus = log.DeviceState.ThermalStatus
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+CURRENT_TAU = 15.   # 15s time constant
+TEMP_TAU = 5.   # 5s time constant
+DISCONNECT_TIMEOUT = 5.  # wait 5 seconds before going offroad after disconnect so you get an alert
+PANDA_STATES_TIMEOUT = round(1000 / SERVICE_LIST['pandaStates'].frequency * 1.5)  # 1.5x the expected pandaState frequency
+
+ThermalBand = namedtuple("ThermalBand", ['min_temp', 'max_temp'])
+HardwareState = namedtuple("HardwareState", ['network_type', 'network_info', 'network_strength', 'network_stats',
+                                             'network_metered', 'nvme_temps', 'modem_temps'])
+
+# List of thermal bands. We will stay within this region as long as we are within the bounds.
+# When exiting the bounds, we'll jump to the lower or higher band. Bands are ordered in the dict.
+THERMAL_BANDS = OrderedDict({
+  ThermalStatus.green: ThermalBand(None, 80.0),
+  ThermalStatus.yellow: ThermalBand(75.0, 96.0),
+  ThermalStatus.red: ThermalBand(88.0, 107.),
+  ThermalStatus.danger: ThermalBand(94.0, None),
+})
+
+# Override to highest thermal band when offroad and above this temp
+OFFROAD_DANGER_TEMP = 75
+
+prev_offroad_states: dict[str, tuple[bool, str | None]] = {}
+
+
+
+def set_offroad_alert_if_changed(offroad_alert: str, show_alert: bool, extra_text: str | None=None):
+  if prev_offroad_states.get(offroad_alert, None) == (show_alert, extra_text):
+    return
+  prev_offroad_states[offroad_alert] = (show_alert, extra_text)
+  set_offroad_alert(offroad_alert, show_alert, extra_text)
+
+def touch_thread(end_event):
+  count = 0
+
+  pm = messaging.PubMaster(["touch"])
+
+  event_format = "llHHi"
+  event_size = struct.calcsize(event_format)
+  event_frame = []
+
+  with open("/dev/input/by-path/platform-894000.i2c-event", "rb") as event_file:
+    fcntl.fcntl(event_file, fcntl.F_SETFL, os.O_NONBLOCK)
+    while not end_event.is_set():
+      if (count % int(1. / DT_HW)) == 0:
+        event = event_file.read(event_size)
+        if event:
+          (sec, usec, etype, code, value) = struct.unpack(event_format, event)
+          if etype != 0 or code != 0 or value != 0:
+            touch = log.Touch.new_message()
+            touch.sec = sec
+            touch.usec = usec
+            touch.type = etype
+            touch.code = code
+            touch.value = value
+            event_frame.append(touch)
+          else: # end of frame, push new log
+            msg = messaging.new_message('touch', len(event_frame), valid=True)
+            msg.touch = event_frame
+            pm.send('touch', msg)
+            event_frame = []
+          continue
+
+      count += 1
+      time.sleep(DT_HW)
+
+
+def hw_state_thread(end_event, hw_queue):
+  """Handles non critical hardware state, and sends over queue"""
+  count = 0
+  prev_hw_state = None
+
+  modem_version = None
+  modem_configured = False
+  modem_restarted = False
+  modem_missing_count = 0
+
+  while not end_event.is_set():
+    # these are expensive calls. update every 10s
+    if (count % int(10. / DT_HW)) == 0:
+      try:
+        network_type = HARDWARE.get_network_type()
+        modem_temps = HARDWARE.get_modem_temperatures()
+        if len(modem_temps) == 0 and prev_hw_state is not None:
+          modem_temps = prev_hw_state.modem_temps
+
+        # Log modem version once
+        if AGNOS and (modem_version is None):
+          modem_version = HARDWARE.get_modem_version()
+
+          if modem_version is not None:
+            cloudlog.event("modem version", version=modem_version)
+          else:
+            if not modem_restarted:
+              # TODO: we may be able to remove this with a MM update
+              # ModemManager's probing on startup can fail
+              # rarely, restart the service to probe again.
+              modem_missing_count += 1
+              if modem_missing_count > 3:
+                modem_restarted = True
+                cloudlog.event("restarting ModemManager")
+                os.system("sudo systemctl restart --no-block ModemManager")
+
+        tx, rx = HARDWARE.get_modem_data_usage()
+
+        hw_state = HardwareState(
+          network_type=network_type,
+          network_info=HARDWARE.get_network_info(),
+          network_strength=HARDWARE.get_network_strength(network_type),
+          network_stats={'wwanTx': tx, 'wwanRx': rx},
+          network_metered=HARDWARE.get_network_metered(network_type),
+          nvme_temps=HARDWARE.get_nvme_temperatures(),
+          modem_temps=modem_temps,
+        )
+
+        try:
+          hw_queue.put_nowait(hw_state)
+        except queue.Full:
+          pass
+
+        if not modem_configured and HARDWARE.get_modem_version() is not None:
+          cloudlog.warning("configuring modem")
+          HARDWARE.configure_modem()
+          modem_configured = True
+
+        prev_hw_state = hw_state
+      except Exception:
+        cloudlog.exception("Error getting hardware state")
+
+    count += 1
+    time.sleep(DT_HW)
+
+from openpilot.system.manager.manager import set_default_params
+def update_restart_condition(current_time, restart_triggered_ts, params, onroad_conditions):
+  if current_time - restart_triggered_ts < 5.:
+    onroad_conditions["not_restart_triggered"] = False
+  else:
+    onroad_conditions["not_restart_triggered"] = True
+    softRestartTriggered = params.get_int("SoftRestartTriggered")
+    if softRestartTriggered > 0:
+      if softRestartTriggered == 2:
+        print("Parameter set to default")
+        set_default_params()
+
+      params.put_int("SoftRestartTriggered", 0)
+      restart_triggered_ts = current_time
+  return restart_triggered_ts
+
+def hardware_thread(end_event, hw_queue) -> None:
+  pm = messaging.PubMaster(['deviceState'])
+  sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates"], poll="pandaStates")
+
+  count = 0
+
+  onroad_conditions: dict[str, bool] = {
+    "ignition": False,
+  }
+  startup_conditions: dict[str, bool] = {}
+  startup_conditions_prev: dict[str, bool] = {}
+
+  off_ts: float | None = None
+  started_ts: float | None = None
+  started_seen = False
+  startup_blocked_ts: float | None = None
+  thermal_status = ThermalStatus.yellow
+
+  last_hw_state = HardwareState(
+    network_type=NetworkType.none,
+    network_info=None,
+    network_metered=False,
+    network_strength=NetworkStrength.unknown,
+    network_stats={'wwanTx': -1, 'wwanRx': -1},
+    nvme_temps=[],
+    modem_temps=[],
+  )
+
+  all_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  offroad_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  should_start_prev = False
+  in_car = False
+  engaged_prev = False
+
+  params = Params()
+  power_monitor = PowerMonitoring()
+
+  HARDWARE.initialize_hardware()
+  thermal_config = HARDWARE.get_thermal_config()
+
+  fan_controller = None
+
+  restart_triggered_ts = 0.
+
+  while not end_event.is_set():
+    sm.update(PANDA_STATES_TIMEOUT)
+
+    pandaStates = sm['pandaStates']
+    peripheralState = sm['peripheralState']
+    peripheral_panda_present = peripheralState.pandaType != log.PandaState.PandaType.unknown
+
+    current_time = time.monotonic()
+    restart_triggered_ts = update_restart_condition(current_time, restart_triggered_ts, params, onroad_conditions)
+
+    if sm.updated['pandaStates'] and len(pandaStates) > 0:
+
+      # Set ignition based on any panda connected
+      onroad_conditions["ignition"] = any(ps.ignitionLine or ps.ignitionCan for ps in pandaStates if ps.pandaType != log.PandaState.PandaType.unknown)
+
+      pandaState = pandaStates[0]
+
+      in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected
+
+      # Setup fan handler on first connect to panda
+      if fan_controller is None and peripheral_panda_present:
+        if TICI:
+          fan_controller = TiciFanController()
+
+    elif (time.monotonic() - sm.recv_time['pandaStates']) > DISCONNECT_TIMEOUT:
+      if onroad_conditions["ignition"]:
+        onroad_conditions["ignition"] = False
+        cloudlog.error("panda timed out onroad")
+
+    # Run at 2Hz, plus either edge of ignition
+    ign_edge = (started_ts is not None) != onroad_conditions["ignition"]
+    if (sm.frame % round(SERVICE_LIST['pandaStates'].frequency * DT_HW) != 0) and not ign_edge:
+      continue
+
+    msg = messaging.new_message('deviceState', valid=True)
+    msg.deviceState = thermal_config.get_msg()
+    msg.deviceState.deviceType = HARDWARE.get_device_type()
+
+    try:
+      last_hw_state = hw_queue.get_nowait()
+    except queue.Empty:
+      pass
+
+    msg.deviceState.freeSpacePercent = get_available_percent(default=100.0)
+    msg.deviceState.memoryUsagePercent = int(round(psutil.virtual_memory().percent))
+    msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent()))
+    online_cpu_usage = [int(round(n)) for n in psutil.cpu_percent(percpu=True)]
+    offline_cpu_usage = [0., ] * (len(msg.deviceState.cpuTempC) - len(online_cpu_usage))
+    msg.deviceState.cpuUsagePercent = online_cpu_usage + offline_cpu_usage
+
+    msg.deviceState.networkType = last_hw_state.network_type
+    msg.deviceState.networkMetered = last_hw_state.network_metered
+    msg.deviceState.networkStrength = last_hw_state.network_strength
+    msg.deviceState.networkStats = last_hw_state.network_stats
+    if last_hw_state.network_info is not None:
+      msg.deviceState.networkInfo = last_hw_state.network_info
+
+    msg.deviceState.nvmeTempC = last_hw_state.nvme_temps
+    msg.deviceState.modemTempC = last_hw_state.modem_temps
+
+    msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness()
+
+    # this subset is only used for offroad
+    temp_sources = [
+      msg.deviceState.memoryTempC,
+      max(msg.deviceState.cpuTempC, default=0.),
+      max(msg.deviceState.gpuTempC, default=0.),
+    ]
+    offroad_comp_temp = offroad_temp_filter.update(max(temp_sources))
+
+    # this drives the thermal status while onroad
+    temp_sources.append(max(msg.deviceState.pmicTempC, default=0.))
+    all_comp_temp = all_temp_filter.update(max(temp_sources))
+    msg.deviceState.maxTempC = all_comp_temp
+
+    if fan_controller is not None:
+      msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
+
+    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
+    if is_offroad_for_5_min and offroad_comp_temp > OFFROAD_DANGER_TEMP:
+      # if device is offroad and already hot without the extra onroad load,
+      # we want to cool down first before increasing load
+      thermal_status = ThermalStatus.danger
+    else:
+      current_band = THERMAL_BANDS[thermal_status]
+      band_idx = list(THERMAL_BANDS.keys()).index(thermal_status)
+      if current_band.min_temp is not None and all_comp_temp < current_band.min_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1]
+      elif current_band.max_temp is not None and all_comp_temp > current_band.max_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1]
+
+    # **** starting logic ****
+
+    startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
+    startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
+    startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version
+
+    # with 2% left, we killall, otherwise the phone will take a long time to boot
+    startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2
+    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version
+    startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled")
+    startup_conditions["not_taking_snapshot"] = not params.get_bool("IsTakingSnapshot")
+
+    # must be at an engageable thermal band to go onroad
+    startup_conditions["device_temp_engageable"] = thermal_status < ThermalStatus.red
+
+    # ensure device is fully booted
+    startup_conditions["device_booted"] = startup_conditions.get("device_booted", False) or HARDWARE.booted()
+
+    # if the temperature enters the danger zone, go offroad to cool down
+    onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger
+    extra_text = f"{offroad_comp_temp:.1f}C"
+    show_alert = (not onroad_conditions["device_temp_good"] or not startup_conditions["device_temp_engageable"]) and onroad_conditions["ignition"]
+    set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", show_alert, extra_text=extra_text)
+
+    # TODO: this should move to TICI.initialize_hardware, but we currently can't import params there
+    if False: #TICI and HARDWARE.get_device_type() == "tici":
+      if not os.path.isfile("/persist/comma/living-in-the-moment"):
+        if not Path("/data/media").is_mount():
+          set_offroad_alert_if_changed("Offroad_StorageMissing", True)
+
+    # Handle offroad/onroad transition
+    should_start = all(onroad_conditions.values())
+    if started_ts is None:
+      should_start = should_start and all(startup_conditions.values())
+
+    if should_start != should_start_prev or (count == 0):
+      params.put_bool("IsEngaged", False)
+      engaged_prev = False
+      HARDWARE.set_power_save(not should_start)
+
+    if sm.updated['selfdriveState']:
+      engaged = sm['selfdriveState'].enabled
+      if engaged != engaged_prev:
+        params.put_bool("IsEngaged", engaged)
+        engaged_prev = engaged
+
+      try:
+        with open('/dev/kmsg', 'w') as kmsg:
+          kmsg.write(f"<3>[hardware] engaged: {engaged}\n")
+      except Exception:
+        pass
+
+    if should_start:
+      off_ts = None
+      if started_ts is None:
+        started_ts = time.monotonic()
+        started_seen = True
+        if startup_blocked_ts is not None:
+          cloudlog.event("Startup after block", block_duration=(time.monotonic() - startup_blocked_ts),
+                         startup_conditions=startup_conditions, onroad_conditions=onroad_conditions,
+                         startup_conditions_prev=startup_conditions_prev, error=True)
+      startup_blocked_ts = None
+    else:
+      if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
+        cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions, error=True)
+        startup_conditions_prev = startup_conditions.copy()
+        startup_blocked_ts = time.monotonic()
+
+      started_ts = None
+      if off_ts is None:
+        off_ts = time.monotonic()
+
+    # Offroad power monitoring
+    voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage
+    power_monitor.calculate(voltage, onroad_conditions["ignition"])
+    msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
+    msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
+    current_power_draw = HARDWARE.get_current_power_draw()
+    statlog.sample("power_draw", current_power_draw)
+    msg.deviceState.powerDrawW = current_power_draw
+
+    som_power_draw = HARDWARE.get_som_power_draw()
+    statlog.sample("som_power_draw", som_power_draw)
+    msg.deviceState.somPowerDrawW = som_power_draw
+
+    # Check if we need to shut down
+    if power_monitor.should_shutdown(onroad_conditions["ignition"], in_car, off_ts, started_seen):
+      cloudlog.warning(f"shutting device down, offroad since {off_ts}")
+      params.put_bool("DoShutdown", True)
+
+    msg.deviceState.started = started_ts is not None
+    msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0))
+
+    last_ping = params.get("LastAthenaPingTime")
+    if last_ping is not None:
+      msg.deviceState.lastAthenaPingTime = int(last_ping)
+
+    msg.deviceState.thermalStatus = thermal_status
+    pm.send("deviceState", msg)
+
+    # Log to statsd
+    statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent)
+    statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent)
+    statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent)
+    for i, usage in enumerate(msg.deviceState.cpuUsagePercent):
+      statlog.gauge(f"cpu{i}_usage_percent", usage)
+    for i, temp in enumerate(msg.deviceState.cpuTempC):
+      statlog.gauge(f"cpu{i}_temperature", temp)
+    for i, temp in enumerate(msg.deviceState.gpuTempC):
+      statlog.gauge(f"gpu{i}_temperature", temp)
+    statlog.gauge("memory_temperature", msg.deviceState.memoryTempC)
+    for i, temp in enumerate(msg.deviceState.pmicTempC):
+      statlog.gauge(f"pmic{i}_temperature", temp)
+    for i, temp in enumerate(last_hw_state.nvme_temps):
+      statlog.gauge(f"nvme_temperature{i}", temp)
+    for i, temp in enumerate(last_hw_state.modem_temps):
+      statlog.gauge(f"modem_temperature{i}", temp)
+    statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired)
+    statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent)
+
+    # report to server once every 10 minutes
+    rising_edge_started = should_start and not should_start_prev
+    if rising_edge_started or (count % int(600. / DT_HW)) == 0:
+      dat = {
+        'count': count,
+        'pandaStates': [strip_deprecated_keys(p.to_dict()) for p in pandaStates],
+        'peripheralState': strip_deprecated_keys(peripheralState.to_dict()),
+        'location': (strip_deprecated_keys(sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None),
+        'deviceState': strip_deprecated_keys(msg.to_dict())
+      }
+      cloudlog.event("STATUS_PACKET", **dat)
+
+      # save last one before going onroad
+      if rising_edge_started:
+        try:
+          params.put("LastOffroadStatusPacket", json.dumps(dat))
+        except Exception:
+          cloudlog.exception("failed to save offroad status")
+
+    params.put_bool_nonblocking("NetworkMetered", msg.deviceState.networkMetered)
+
+    count += 1
+    should_start_prev = should_start
+
+
+def main():
+  hw_queue = queue.Queue(maxsize=1)
+  end_event = threading.Event()
+
+  threads = [
+    threading.Thread(target=hw_state_thread, args=(end_event, hw_queue)),
+    threading.Thread(target=hardware_thread, args=(end_event, hw_queue)),
+  ]
+
+  if TICI:
+    threads.append(threading.Thread(target=touch_thread, args=(end_event,)))
+
+  for t in threads:
+    t.start()
+
+  try:
+    while True:
+      time.sleep(1)
+      if not all(t.is_alive() for t in threads):
+        break
+  finally:
+    end_event.set()
+
+  for t in threads:
+    t.join()
+
+
+if __name__ == "__main__":
+  main()

system/hardware/hw.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <string>
+
+#include "system/hardware/base.h"
+#include "common/util.h"
+
+#if QCOM2
+#include "system/hardware/tici/hardware.h"
+#define Hardware HardwareTici
+#else
+#include "system/hardware/pc/hardware.h"
+#define Hardware HardwarePC
+#endif
+
+namespace Path {
+  inline std::string openpilot_prefix() {
+    return util::getenv("OPENPILOT_PREFIX", "");
+  }
+
+  inline std::string comma_home() {
+    return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix();
+  }
+
+  inline std::string log_root() {
+    if (const char *env = getenv("LOG_ROOT")) {
+      return env;
+    }
+    return Hardware::PC() ? Path::comma_home() + "/media/0/realdata" : "/data/media/0/realdata";
+  }
+
+  inline std::string params() {
+    return util::getenv("PARAMS_ROOT", Hardware::PC() ? (Path::comma_home() + "/params") : "/data/params");
+  }
+
+  inline std::string rsa_file() {
+    return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa";
+  }
+
+  inline std::string swaglog_ipc() {
+    return "ipc:///tmp/logmessage" + Path::openpilot_prefix();
+  }
+
+  inline std::string download_cache_root() {
+    if (const char *env = getenv("COMMA_CACHE")) {
+      return env;
+    }
+    return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/";
+  }
+
+ inline std::string shm_path() {
+    #ifdef __APPLE__
+     return"/tmp";
+    #else
+     return "/dev/shm";
+    #endif
+ }
+}  // namespace Path

system/hardware/hw.py

@@ -0,0 +1,65 @@
+import os
+import platform
+from pathlib import Path
+
+from openpilot.system.hardware import PC
+
+DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
+
+class Paths:
+  @staticmethod
+  def comma_home() -> str:
+    return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", ""))
+
+  @staticmethod
+  def log_root() -> str:
+    if os.environ.get('LOG_ROOT', False):
+      return os.environ['LOG_ROOT']
+    elif PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "realdata")
+    else:
+      return '/data/media/0/realdata/'
+
+  @staticmethod
+  def swaglog_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "log")
+    else:
+      return "/data/log/"
+
+  @staticmethod
+  def swaglog_ipc() -> str:
+    return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "")
+
+  @staticmethod
+  def download_cache_root() -> str:
+    if os.environ.get('COMMA_CACHE', False):
+      return os.environ['COMMA_CACHE'] + "/"
+    return DEFAULT_DOWNLOAD_CACHE_ROOT + os.environ.get("OPENPILOT_PREFIX", "") + "/"
+
+  @staticmethod
+  def persist_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "persist")
+    else:
+      return "/persist/"
+
+  @staticmethod
+  def stats_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "stats")
+    else:
+      return "/data/stats/"
+
+  @staticmethod
+  def config_root() -> str:
+    if PC:
+      return Paths.comma_home()
+    else:
+      return "/tmp/.comma"
+
+  @staticmethod
+  def shm_path() -> str:
+    if PC and platform.system() == "Darwin":
+      return "/tmp"  # This is not really shared memory on macOS, but it's the closest we can get
+    return "/dev/shm"

system/hardware/pc/hardware.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <string>
+
+#include "system/hardware/base.h"
+
+class HardwarePC : public HardwareNone {
+public:
+  static std::string get_os_version() { return "openpilot for PC"; }
+  static std::string get_name() { return "pc"; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
+  static bool PC() { return true; }
+  static bool TICI() { return util::getenv("TICI", 0) == 1; }
+  static bool AGNOS() { return util::getenv("TICI", 0) == 1; }
+
+  static void config_cpu_rendering(bool offscreen) {
+    if (offscreen) {
+      setenv("QT_QPA_PLATFORM", "offscreen", 1);
+    }
+    setenv("__GLX_VENDOR_LIBRARY_NAME", "mesa", 1);
+    setenv("LP_NUM_THREADS", "0", 1); // disable threading so we stay on our assigned CPU
+  }
+};

system/hardware/pc/hardware.py

@@ -0,0 +1,80 @@
+import random
+
+from cereal import log
+from openpilot.system.hardware.base import HardwareBase, LPABase
+
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+
+class Pc(HardwareBase):
+  def get_os_version(self):
+    return None
+
+  def get_device_type(self):
+    return "pc"
+
+  def reboot(self, reason=None):
+    print("REBOOT!")
+
+  def uninstall(self):
+    print("uninstall")
+
+  def get_imei(self, slot):
+    return f"{random.randint(0, 1 << 32):015d}"
+
+  def get_serial(self):
+    return "cccccccc"
+
+  def get_network_info(self):
+    return None
+
+  def get_network_type(self):
+    return NetworkType.wifi
+
+  def get_sim_info(self):
+    return {
+      'sim_id': '',
+      'mcc_mnc': None,
+      'network_type': ["Unknown"],
+      'sim_state': ["ABSENT"],
+      'data_connected': False
+    }
+
+  def get_sim_lpa(self) -> LPABase:
+    raise NotImplementedError("SIM LPA not implemented for PC")
+
+  def get_network_strength(self, network_type):
+    return NetworkStrength.unknown
+
+  def get_current_power_draw(self):
+    return 0
+
+  def get_som_power_draw(self):
+    return 0
+
+  def shutdown(self):
+    print("SHUTDOWN!")
+
+  def set_screen_brightness(self, percentage):
+    pass
+
+  def get_screen_brightness(self):
+    return 0
+
+  def set_power_save(self, powersave_enabled):
+    pass
+
+  def get_gpu_usage_percent(self):
+    return 0
+
+  def get_modem_temperatures(self):
+    return []
+
+  def get_nvme_temperatures(self):
+    return []
+
+  def initialize_hardware(self):
+    pass
+
+  def get_networks(self):
+    return None

system/hardware/power_monitoring.py

@@ -0,0 +1,129 @@
+import time
+import threading
+
+from openpilot.common.params import Params
+from openpilot.system.hardware import HARDWARE
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.statsd import statlog
+
+CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1))
+
+# While driving, a battery charges completely in about 30-60 minutes
+CAR_BATTERY_CAPACITY_uWh = 30e6
+CAR_CHARGING_RATE_W = 45
+
+VBATT_PAUSE_CHARGING = 11.8           # Lower limit on the LPF car battery voltage
+MAX_TIME_OFFROAD_S = 30*3600
+MIN_ON_TIME_S = 3600
+DELAY_SHUTDOWN_TIME_S = 300 # Wait at least DELAY_SHUTDOWN_TIME_S seconds after offroad_time to shutdown.
+VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 60
+
+class PowerMonitoring:
+  def __init__(self):
+    self.params = Params()
+    self.last_measurement_time = None           # Used for integration delta
+    self.last_save_time = 0                     # Used for saving current value in a param
+    self.power_used_uWh = 0                     # Integrated power usage in uWh since going into offroad
+    self.next_pulsed_measurement_time = None
+    self.car_voltage_mV = 12e3                  # Low-passed version of peripheralState voltage
+    self.car_voltage_instant_mV = 12e3          # Last value of peripheralState voltage
+    self.integration_lock = threading.Lock()
+
+    car_battery_capacity_uWh = self.params.get("CarBatteryCapacity")
+    if car_battery_capacity_uWh is None:
+      car_battery_capacity_uWh = 0
+
+    # Reset capacity if it's low
+    self.car_battery_capacity_uWh = max((CAR_BATTERY_CAPACITY_uWh / 10), int(car_battery_capacity_uWh))
+
+  # Calculation tick
+  def calculate(self, voltage: int | None, ignition: bool):
+    try:
+      now = time.monotonic()
+
+      # If peripheralState is None, we're probably not in a car, so we don't care
+      if voltage is None:
+        with self.integration_lock:
+          self.last_measurement_time = None
+          self.next_pulsed_measurement_time = None
+          self.power_used_uWh = 0
+        return
+
+      # Low-pass battery voltage
+      self.car_voltage_instant_mV = voltage
+      self.car_voltage_mV = ((voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))
+      statlog.gauge("car_voltage", self.car_voltage_mV / 1e3)
+
+      # Cap the car battery power and save it in a param every 10-ish seconds
+      self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0)
+      self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh)
+      if now - self.last_save_time >= 10:
+        self.params.put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh)))
+        self.last_save_time = now
+
+      # First measurement, set integration time
+      with self.integration_lock:
+        if self.last_measurement_time is None:
+          self.last_measurement_time = now
+          return
+
+      if ignition:
+        # If there is ignition, we integrate the charging rate of the car
+        with self.integration_lock:
+          self.power_used_uWh = 0
+          integration_time_h = (now - self.last_measurement_time) / 3600
+          if integration_time_h < 0:
+            raise ValueError(f"Negative integration time: {integration_time_h}h")
+          self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h)
+          self.last_measurement_time = now
+      else:
+        # Get current power draw somehow
+        current_power = HARDWARE.get_current_power_draw()
+
+        # Do the integration
+        self._perform_integration(now, current_power)
+    except Exception:
+      cloudlog.exception("Power monitoring calculation failed")
+
+  def _perform_integration(self, t: float, current_power: float) -> None:
+    with self.integration_lock:
+      try:
+        if self.last_measurement_time:
+          integration_time_h = (t - self.last_measurement_time) / 3600
+          power_used = (current_power * 1000000) * integration_time_h
+          if power_used < 0:
+            raise ValueError(f"Negative power used! Integration time: {integration_time_h} h Current Power: {power_used} uWh")
+          self.power_used_uWh += power_used
+          self.car_battery_capacity_uWh -= power_used
+          self.last_measurement_time = t
+      except Exception:
+        cloudlog.exception("Integration failed")
+
+  # Get the power usage
+  def get_power_used(self) -> int:
+    return int(self.power_used_uWh)
+
+  def get_car_battery_capacity(self) -> int:
+    return int(self.car_battery_capacity_uWh)
+
+  # See if we need to shutdown
+  def should_shutdown(self, ignition: bool, in_car: bool, offroad_timestamp: float | None, started_seen: bool):
+    if offroad_timestamp is None:
+      return False
+
+    now = time.monotonic()
+    should_shutdown = False
+    offroad_time = (now - offroad_timestamp)
+    low_voltage_shutdown = (self.car_voltage_mV < (VBATT_PAUSE_CHARGING * 1e3) and
+                            offroad_time > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S)
+    MAX_TIME_OFFROAD_S = Params().get_int("MaxTimeOffroadMin") * 60
+    should_shutdown |= offroad_time > MAX_TIME_OFFROAD_S
+    should_shutdown |= low_voltage_shutdown
+    should_shutdown |= (self.car_battery_capacity_uWh <= 0)
+    should_shutdown &= not ignition
+    should_shutdown &= (not self.params.get_bool("DisablePowerDown"))
+    should_shutdown &= in_car
+    should_shutdown &= offroad_time > DELAY_SHUTDOWN_TIME_S
+    should_shutdown |= self.params.get_bool("ForcePowerDown")
+    should_shutdown &= started_seen or (now > MIN_ON_TIME_S)
+    return should_shutdown

system/hardware/tests/test_fan_controller.py

@@ -0,0 +1,50 @@
+import pytest
+
+from openpilot.system.hardware.fan_controller import TiciFanController
+
+ALL_CONTROLLERS = [TiciFanController]
+
+def patched_controller(mocker, controller_class):
+  mocker.patch("os.system", new=mocker.Mock())
+  return controller_class()
+
+class TestFanController:
+  def wind_up(self, controller, ignition=True):
+    for _ in range(1000):
+      controller.update(100, ignition)
+
+  def wind_down(self, controller, ignition=False):
+    for _ in range(1000):
+      controller.update(10, ignition)
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_hot_onroad(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller)
+    assert controller.update(100, True) >= 70
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_offroad_limits(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller)
+    assert controller.update(100, False) <= 30
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_no_fan_wear(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_down(controller)
+    assert controller.update(10, False) == 0
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_limited(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller, True)
+    assert controller.update(100, True) == 100
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_windup_speed(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_down(controller, True)
+    for _ in range(10):
+      controller.update(90, True)
+    assert controller.update(90, True) >= 60

system/hardware/tests/test_power_monitoring.py

@@ -0,0 +1,199 @@
+import pytest
+
+from openpilot.common.params import Params
+from openpilot.system.hardware.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \
+                                                CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S
+
+# Create fake time
+ssb = 0.
+def mock_time_monotonic():
+  global ssb
+  ssb += 1.
+  return ssb
+
+TEST_DURATION_S = 50
+GOOD_VOLTAGE = 12 * 1e3
+VOLTAGE_BELOW_PAUSE_CHARGING = (VBATT_PAUSE_CHARGING - 1) * 1e3
+
+def pm_patch(mocker, name, value, constant=False):
+  if constant:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", value)
+  else:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", return_value=value)
+
+
+@pytest.fixture(autouse=True)
+def mock_time(mocker):
+  mocker.patch("time.monotonic", mock_time_monotonic)
+
+
+class TestPowerMonitoring:
+  def setup_method(self):
+    self.params = Params()
+
+  # Test to see that it doesn't do anything when pandaState is None
+  def test_panda_state_present(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(None, None)
+    assert pm.get_power_used() == 0
+    assert pm.get_car_battery_capacity() == (CAR_BATTERY_CAPACITY_uWh / 10)
+
+  # Test to see that it doesn't integrate offroad when ignition is True
+  def test_offroad_ignition(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(GOOD_VOLTAGE, True)
+    assert pm.get_power_used() == 0
+
+  # Test to see that it integrates with discharging battery
+  def test_offroad_integration_discharging(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_power_usage = ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_power_used() - expected_power_usage) < 10
+
+  # Test to check positive integration of car_battery_capacity
+  def test_car_battery_integration_onroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    expected_capacity = ((TEST_DURATION_S/3600) * CAR_CHARGING_RATE_W * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check positive integration upper limit
+  def test_car_battery_integration_upper_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh - 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    estimated_capacity = CAR_BATTERY_CAPACITY_uWh + (CAR_CHARGING_RATE_W / 3600 * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check negative integration of car_battery_capacity
+  def test_car_battery_integration_offroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_capacity = CAR_BATTERY_CAPACITY_uWh - ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check negative integration lower limit
+  def test_car_battery_integration_lower_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    estimated_capacity = 0 - ((1/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check policy of stopping charging after MAX_TIME_OFFROAD_S
+  def test_max_time_offroad(self, mocker):
+    MOCKED_MAX_OFFROAD_TIME = 3600
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    pm_patch(mocker, "MAX_TIME_OFFROAD_S", MOCKED_MAX_OFFROAD_TIME, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    start_time = ssb
+    ignition = False
+    while ssb <= start_time + MOCKED_MAX_OFFROAD_TIME:
+      pm.calculate(GOOD_VOLTAGE, ignition)
+      if (ssb - start_time) % 1000 == 0 and ssb < start_time + MOCKED_MAX_OFFROAD_TIME:
+        assert not pm.should_shutdown(ignition, True, start_time, False)
+    assert pm.should_shutdown(ignition, True, start_time, False)
+
+  def test_car_voltage(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 350
+    VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 50
+    pm_patch(mocker, "VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S", VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    start_time = ssb
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert pm.should_shutdown(ignition, True, start_time, True) == \
+                          (pm.car_voltage_mV < VBATT_PAUSE_CHARGING * 1e3 and \
+                          (ssb - start_time) > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S and \
+                            (ssb - start_time) > DELAY_SHUTDOWN_TIME_S)
+    assert pm.should_shutdown(ignition, True, start_time, True)
+
+  # Test to check policy of not stopping charging when DisablePowerDown is set
+  def test_disable_power_down(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    self.params.put_bool("DisablePowerDown", True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when ignition
+  def test_ignition(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = True
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when harness is not connected
+  def test_harness_connection(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, False, ssb, False)
+    assert not pm.should_shutdown(ignition, False, ssb, False)
+
+  def test_delay_shutdown_time(self):
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    ignition = False
+    in_car = True
+    offroad_timestamp = ssb
+    started_seen = True
+    pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+
+    while ssb < offroad_timestamp + DELAY_SHUTDOWN_TIME_S:
+      assert not pm.should_shutdown(ignition, in_car,
+                                          offroad_timestamp,
+                                          started_seen), \
+                       f"Should not shutdown before {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
+    assert pm.should_shutdown(ignition, in_car,
+                                       offroad_timestamp,
+                                       started_seen), \
+                    f"Should shutdown after {DELAY_SHUTDOWN_TIME_S} seconds offroad time"

system/hardware/tici/agnos.json

@@ -0,0 +1,84 @@
+[
+  {
+    "name": "xbl",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl-6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1.img.xz",
+    "hash": "6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1",
+    "hash_raw": "6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1",
+    "size": 3282256,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "003a17ab1be68a696f7efe4c9938e8be511d4aacfc2f3211fc896bdc1681d089"
+  },
+  {
+    "name": "xbl_config",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535.img.xz",
+    "hash": "63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535",
+    "hash_raw": "63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535",
+    "size": 98124,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "2a855dd636cc94718b64bea83a44d0a31741ecaa8f72a63613ff348ec7404091"
+  },
+  {
+    "name": "abl",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6.img.xz",
+    "hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
+    "hash_raw": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
+    "size": 274432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6"
+  },
+  {
+    "name": "aop",
+    "url": "https://commadist.azureedge.net/agnosupdate/aop-21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9.img.xz",
+    "hash": "21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9",
+    "hash_raw": "21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9",
+    "size": 184364,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "c1be2f4aac5b3af49b904b027faec418d05efd7bd5144eb4fdfcba602bcf2180"
+  },
+  {
+    "name": "devcfg",
+    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620.img.xz",
+    "hash": "d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620",
+    "hash_raw": "d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620",
+    "size": 40336,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "17b229668b20305ff8fa3cd5f94716a3aaa1e5bf9d1c24117eff7f2f81ae719f"
+  },
+  {
+    "name": "boot",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef.img.xz",
+    "hash": "4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef",
+    "hash_raw": "4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef",
+    "size": 18479104,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "8d7094d774faa4e801e36b403a31b53b913b31d086f4dc682d2f64710c557e8a"
+  },
+  {
+    "name": "system",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39.img.xz",
+    "hash": "cccd7073d067027396f2afd49874729757db0bbbc79853a0bf2938bd356fe164",
+    "hash_raw": "4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39",
+    "size": 5368709120,
+    "sparse": true,
+    "full_check": false,
+    "has_ab": true,
+    "ondevice_hash": "c7707f16ce7d977748677cc354e250943b4ff6c21b9a19a492053d32397cf9ec",
+    "alt": {
+      "hash": "4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39.img",
+      "size": 5368709120
+    }
+  }
+]

system/hardware/tici/agnos.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+import hashlib
+import json
+import lzma
+import os
+import struct
+import subprocess
+import time
+from collections.abc import Generator
+
+import requests
+
+import openpilot.system.updated.casync.casync as casync
+
+SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
+CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"
+
+AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
+
+
+class StreamingDecompressor:
+  def __init__(self, url: str) -> None:
+    self.buf = b""
+
+    self.req = requests.get(url, stream=True, headers={'Accept-Encoding': None}, timeout=60)
+    self.it = self.req.iter_content(chunk_size=1024 * 1024)
+    self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
+    self.eof = False
+    self.sha256 = hashlib.sha256()
+
+  def read(self, length: int) -> bytes:
+    while len(self.buf) < length and not self.eof:
+      if self.decompressor.needs_input:
+        self.req.raise_for_status()
+
+        try:
+          compressed = next(self.it)
+        except StopIteration:
+          self.eof = True
+          break
+      else:
+        compressed = b''
+
+      self.buf += self.decompressor.decompress(compressed, max_length=length)
+
+      if self.decompressor.eof:
+        self.eof = True
+        break
+
+    result = self.buf[:length]
+    self.buf = self.buf[length:]
+
+    self.sha256.update(result)
+    return result
+
+
+def unsparsify(f: StreamingDecompressor) -> Generator[bytes, None, None]:
+  # https://source.android.com/devices/bootloader/images#sparse-format
+  magic = struct.unpack("I", f.read(4))[0]
+  assert(magic == 0xed26ff3a)
+
+  # Version
+  major = struct.unpack("H", f.read(2))[0]
+  minor = struct.unpack("H", f.read(2))[0]
+  assert(major == 1 and minor == 0)
+
+  f.read(2)  # file header size
+  f.read(2)  # chunk header size
+
+  block_sz = struct.unpack("I", f.read(4))[0]
+  f.read(4)  # total blocks
+  num_chunks = struct.unpack("I", f.read(4))[0]
+  f.read(4)  # crc checksum
+
+  for _ in range(num_chunks):
+    chunk_type, out_blocks = SPARSE_CHUNK_FMT.unpack(f.read(12))
+
+    if chunk_type == 0xcac1:  # Raw
+      # TODO: yield in smaller chunks. Yielding only block_sz is too slow. Largest observed data chunk is 252 MB.
+      yield f.read(out_blocks * block_sz)
+    elif chunk_type == 0xcac2:  # Fill
+      filler = f.read(4) * (block_sz // 4)
+      for _ in range(out_blocks):
+        yield filler
+    elif chunk_type == 0xcac3:  # Don't care
+      yield b""
+    else:
+      raise Exception("Unhandled sparse chunk type")
+
+
+# noop wrapper with same API as unsparsify() for non sparse images
+def noop(f: StreamingDecompressor) -> Generator[bytes, None, None]:
+  while len(chunk := f.read(1024 * 1024)) > 0:
+    yield chunk
+
+
+def get_target_slot_number() -> int:
+  current_slot = subprocess.check_output(["abctl", "--boot_slot"], encoding='utf-8').strip()
+  return 1 if current_slot == "_a" else 0
+
+
+def slot_number_to_suffix(slot_number: int) -> str:
+  assert slot_number in (0, 1)
+  return '_a' if slot_number == 0 else '_b'
+
+
+def get_partition_path(target_slot_number: int, partition: dict) -> str:
+  path = f"/dev/disk/by-partlabel/{partition['name']}"
+
+  if partition.get('has_ab', True):
+    path += slot_number_to_suffix(target_slot_number)
+
+  return path
+
+
+def get_raw_hash(path: str, partition_size: int) -> str:
+  raw_hash = hashlib.sha256()
+  pos, chunk_size = 0, 1024 * 1024
+
+  with open(path, 'rb+') as out:
+    while pos < partition_size:
+      n = min(chunk_size, partition_size - pos)
+      raw_hash.update(out.read(n))
+      pos += n
+
+  return raw_hash.hexdigest().lower()
+
+
+def verify_partition(target_slot_number: int, partition: dict[str, str | int], force_full_check: bool = False) -> bool:
+  full_check = partition['full_check'] or force_full_check
+  path = get_partition_path(target_slot_number, partition)
+
+  if not isinstance(partition['size'], int):
+    return False
+
+  partition_size: int = partition['size']
+
+  if not isinstance(partition['hash_raw'], str):
+    return False
+
+  partition_hash: str = partition['hash_raw']
+
+  if full_check:
+    return get_raw_hash(path, partition_size) == partition_hash.lower()
+  else:
+    with open(path, 'rb+') as out:
+      out.seek(partition_size)
+      return out.read(64) == partition_hash.lower().encode()
+
+
+def clear_partition_hash(target_slot_number: int, partition: dict) -> None:
+  path = get_partition_path(target_slot_number, partition)
+  with open(path, 'wb+') as out:
+    partition_size = partition['size']
+
+    out.seek(partition_size)
+    out.write(b"\x00" * 64)
+    os.sync()
+
+
+def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog):
+  path = get_partition_path(target_slot_number, partition)
+  downloader = StreamingDecompressor(partition['url'])
+
+  with open(path, 'wb+') as out:
+    # Flash partition
+    last_p = 0
+    raw_hash = hashlib.sha256()
+    f = unsparsify if partition['sparse'] else noop
+    for chunk in f(downloader):
+      raw_hash.update(chunk)
+      out.write(chunk)
+      p = int(out.tell() / partition['size'] * 100)
+      if p != last_p:
+        last_p = p
+        print(f"Installing {partition['name']}: {p}", flush=True)
+
+    if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
+      raise Exception(f"Raw hash mismatch '{raw_hash.hexdigest().lower()}'")
+
+    if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
+      raise Exception("Uncompressed hash mismatch")
+
+    if out.tell() != partition['size']:
+      raise Exception("Uncompressed size mismatch")
+
+    os.sync()
+
+
+def extract_casync_image(target_slot_number: int, partition: dict, cloudlog):
+  path = get_partition_path(target_slot_number, partition)
+  seed_path = path[:-1] + ('b' if path[-1] == 'a' else 'a')
+
+  target = casync.parse_caibx(partition['casync_caibx'])
+
+  sources: list[tuple[str, casync.ChunkReader, casync.ChunkDict]] = []
+
+  # First source is the current partition.
+  try:
+    raw_hash = get_raw_hash(seed_path, partition['size'])
+    caibx_url = f"{CAIBX_URL}{partition['name']}-{raw_hash}.caibx"
+
+    try:
+      cloudlog.info(f"casync fetching {caibx_url}")
+      sources += [('seed', casync.FileChunkReader(seed_path), casync.build_chunk_dict(casync.parse_caibx(caibx_url)))]
+    except requests.RequestException:
+      cloudlog.error(f"casync failed to load {caibx_url}")
+  except Exception:
+    cloudlog.exception("casync failed to hash seed partition")
+
+  # Second source is the target partition, this allows for resuming
+  sources += [('target', casync.FileChunkReader(path), casync.build_chunk_dict(target))]
+
+  # Finally we add the remote source to download any missing chunks
+  sources += [('remote', casync.RemoteChunkReader(partition['casync_store']), casync.build_chunk_dict(target))]
+
+  last_p = 0
+
+  def progress(cur):
+    nonlocal last_p
+    p = int(cur / partition['size'] * 100)
+    if p != last_p:
+      last_p = p
+      print(f"Installing {partition['name']}: {p}", flush=True)
+
+  stats = casync.extract(target, sources, path, progress)
+  cloudlog.error(f'casync done {json.dumps(stats)}')
+
+  os.sync()
+  if not verify_partition(target_slot_number, partition, force_full_check=True):
+    raise Exception(f"Raw hash mismatch '{partition['hash_raw'].lower()}'")
+
+
+def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False):
+  cloudlog.info(f"Downloading and writing {partition['name']}")
+
+  if verify_partition(target_slot_number, partition):
+    cloudlog.info(f"Already flashed {partition['name']}")
+    return
+
+  # Clear hash before flashing in case we get interrupted
+  full_check = partition['full_check']
+  if not full_check:
+    clear_partition_hash(target_slot_number, partition)
+
+  path = get_partition_path(target_slot_number, partition)
+
+  if ('casync_caibx' in partition) and not standalone:
+    extract_casync_image(target_slot_number, partition, cloudlog)
+  else:
+    extract_compressed_image(target_slot_number, partition, cloudlog)
+
+  # Write hash after successful flash
+  if not full_check:
+    with open(path, 'wb+') as out:
+      out.seek(partition['size'])
+      out.write(partition['hash_raw'].lower().encode())
+
+
+def swap(manifest_path: str, target_slot_number: int, cloudlog) -> None:
+  update = json.load(open(manifest_path))
+  for partition in update:
+    if not partition.get('full_check', False):
+      clear_partition_hash(target_slot_number, partition)
+
+  while True:
+    out = subprocess.check_output(f"abctl --set_active {target_slot_number}", shell=True, stderr=subprocess.STDOUT, encoding='utf8')
+    if ("No such file or directory" not in out) and ("lun as boot lun" in out):
+      cloudlog.info(f"Swap successful {out}")
+      break
+    else:
+      cloudlog.error(f"Swap failed {out}")
+
+
+def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, standalone=False) -> None:
+  update = json.load(open(manifest_path))
+
+  cloudlog.info(f"Target slot {target_slot_number}")
+
+  # set target slot as unbootable
+  os.system(f"abctl --set_unbootable {target_slot_number}")
+
+  for partition in update:
+    success = False
+
+    for retries in range(10):
+      try:
+        flash_partition(target_slot_number, partition, cloudlog, standalone)
+        success = True
+        break
+
+      except requests.exceptions.RequestException:
+        cloudlog.exception("Failed")
+        cloudlog.info(f"Failed to download {partition['name']}, retrying ({retries})")
+        time.sleep(10)
+
+    if not success:
+      cloudlog.info(f"Failed to flash {partition['name']}, aborting")
+      raise Exception("Maximum retries exceeded")
+
+  cloudlog.info(f"AGNOS ready on slot {target_slot_number}")
+
+
+def verify_agnos_update(manifest_path: str, target_slot_number: int) -> bool:
+  update = json.load(open(manifest_path))
+  return all(verify_partition(target_slot_number, partition) for partition in update)
+
+
+if __name__ == "__main__":
+  import argparse
+  import logging
+
+  parser = argparse.ArgumentParser(description="Flash and verify AGNOS update",
+                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+  parser.add_argument("--verify", action="store_true", help="Verify and perform swap if update ready")
+  parser.add_argument("--swap", action="store_true", help="Verify and perform swap, downloads if necessary")
+  parser.add_argument("manifest", help="Manifest json")
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.INFO)
+
+  target_slot_number = get_target_slot_number()
+  if args.verify:
+    if verify_agnos_update(args.manifest, target_slot_number):
+      swap(args.manifest, target_slot_number, logging)
+      exit(0)
+    exit(1)
+  elif args.swap:
+    while not verify_agnos_update(args.manifest, target_slot_number):
+      logging.error("Verification failed. Flashing AGNOS")
+      flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)
+
+    logging.warning(f"Verification succeeded. Swapping to slot {target_slot_number}")
+    swap(args.manifest, target_slot_number, logging)
+  else:
+    flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)

system/hardware/tici/all-partitions.json

@@ -0,0 +1,400 @@
+[
+  {
+    "name": "gpt_main_0",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_0-8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd.img.xz",
+    "hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "hash_raw": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "gpt": {
+      "lun": 0,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_1",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_1-fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6.img.xz",
+    "hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "hash_raw": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "gpt": {
+      "lun": 1,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_2",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_2-5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21.img.xz",
+    "hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "hash_raw": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "gpt": {
+      "lun": 2,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_3",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_3-c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159.img.xz",
+    "hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "hash_raw": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "gpt": {
+      "lun": 3,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_4",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_4-e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e.img.xz",
+    "hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "hash_raw": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "gpt": {
+      "lun": 4,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_5",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_5-21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3.img.xz",
+    "hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "hash_raw": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "gpt": {
+      "lun": 5,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "persist",
+    "url": "https://commadist.azureedge.net/agnosupdate/persist-9814b07851292f510f3794b767489f38ab379a99f0ea75dc620ad2d3a496d54d.img.xz",
+    "hash": "9814b07851292f510f3794b767489f38ab379a99f0ea75dc620ad2d3a496d54d",
+    "hash_raw": "9814b07851292f510f3794b767489f38ab379a99f0ea75dc620ad2d3a496d54d",
+    "size": 33554432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "9814b07851292f510f3794b767489f38ab379a99f0ea75dc620ad2d3a496d54d"
+  },
+  {
+    "name": "systemrw",
+    "url": "https://commadist.azureedge.net/agnosupdate/systemrw-8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e.img.xz",
+    "hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
+    "hash_raw": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
+    "size": 16777216,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e"
+  },
+  {
+    "name": "cache",
+    "url": "https://commadist.azureedge.net/agnosupdate/cache-ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4.img.xz",
+    "hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
+    "hash_raw": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
+    "size": 134217728,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4"
+  },
+  {
+    "name": "xbl",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl-6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1.img.xz",
+    "hash": "6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1",
+    "hash_raw": "6710967ca9701f205d7ab19c3a9b0dd2f547e65b3d96048b7c2b03755aafa0f1",
+    "size": 3282256,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "003a17ab1be68a696f7efe4c9938e8be511d4aacfc2f3211fc896bdc1681d089"
+  },
+  {
+    "name": "xbl_config",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535.img.xz",
+    "hash": "63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535",
+    "hash_raw": "63922cfbfdf4ab87986c4ba8f3a4df5bf28414b3f71a29ec5947336722215535",
+    "size": 98124,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "2a855dd636cc94718b64bea83a44d0a31741ecaa8f72a63613ff348ec7404091"
+  },
+  {
+    "name": "abl",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6.img.xz",
+    "hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
+    "hash_raw": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
+    "size": 274432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6"
+  },
+  {
+    "name": "aop",
+    "url": "https://commadist.azureedge.net/agnosupdate/aop-21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9.img.xz",
+    "hash": "21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9",
+    "hash_raw": "21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9",
+    "size": 184364,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "c1be2f4aac5b3af49b904b027faec418d05efd7bd5144eb4fdfcba602bcf2180"
+  },
+  {
+    "name": "bluetooth",
+    "url": "https://commadist.azureedge.net/agnosupdate/bluetooth-9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533.img.xz",
+    "hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
+    "hash_raw": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
+    "size": 1048576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533"
+  },
+  {
+    "name": "cmnlib64",
+    "url": "https://commadist.azureedge.net/agnosupdate/cmnlib64-1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3.img.xz",
+    "hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
+    "hash_raw": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3"
+  },
+  {
+    "name": "cmnlib",
+    "url": "https://commadist.azureedge.net/agnosupdate/cmnlib-63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82.img.xz",
+    "hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
+    "hash_raw": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82"
+  },
+  {
+    "name": "devcfg",
+    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620.img.xz",
+    "hash": "d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620",
+    "hash_raw": "d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620",
+    "size": 40336,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "17b229668b20305ff8fa3cd5f94716a3aaa1e5bf9d1c24117eff7f2f81ae719f"
+  },
+  {
+    "name": "devinfo",
+    "url": "https://commadist.azureedge.net/agnosupdate/devinfo-143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3.img.xz",
+    "hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
+    "hash_raw": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
+    "size": 4096,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3"
+  },
+  {
+    "name": "dsp",
+    "url": "https://commadist.azureedge.net/agnosupdate/dsp-4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248.img.xz",
+    "hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
+    "hash_raw": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
+    "size": 33554432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248"
+  },
+  {
+    "name": "hyp",
+    "url": "https://commadist.azureedge.net/agnosupdate/hyp-ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927.img.xz",
+    "hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
+    "hash_raw": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927"
+  },
+  {
+    "name": "keymaster",
+    "url": "https://commadist.azureedge.net/agnosupdate/keymaster-5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04.img.xz",
+    "hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
+    "hash_raw": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04"
+  },
+  {
+    "name": "limits",
+    "url": "https://commadist.azureedge.net/agnosupdate/limits-94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1.img.xz",
+    "hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
+    "hash_raw": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
+    "size": 4096,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1"
+  },
+  {
+    "name": "logfs",
+    "url": "https://commadist.azureedge.net/agnosupdate/logfs-b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220.img.xz",
+    "hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
+    "hash_raw": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
+    "size": 8388608,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220"
+  },
+  {
+    "name": "modem",
+    "url": "https://commadist.azureedge.net/agnosupdate/modem-a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994.img.xz",
+    "hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
+    "hash_raw": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
+    "size": 125829120,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994"
+  },
+  {
+    "name": "qupfw",
+    "url": "https://commadist.azureedge.net/agnosupdate/qupfw-64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a.img.xz",
+    "hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
+    "hash_raw": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
+    "size": 65536,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a"
+  },
+  {
+    "name": "splash",
+    "url": "https://commadist.azureedge.net/agnosupdate/splash-5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08.img.xz",
+    "hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
+    "hash_raw": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
+    "size": 34226176,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08"
+  },
+  {
+    "name": "storsec",
+    "url": "https://commadist.azureedge.net/agnosupdate/storsec-4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce.img.xz",
+    "hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
+    "hash_raw": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
+    "size": 131072,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce"
+  },
+  {
+    "name": "tz",
+    "url": "https://commadist.azureedge.net/agnosupdate/tz-e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16.img.xz",
+    "hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
+    "hash_raw": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
+    "size": 2097152,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16"
+  },
+  {
+    "name": "boot",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef.img.xz",
+    "hash": "4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef",
+    "hash_raw": "4de8f892dbac3fa3fee1efe68ca76e23e75812e81a6577d00d52e2da1ef624ef",
+    "size": 18479104,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "8d7094d774faa4e801e36b403a31b53b913b31d086f4dc682d2f64710c557e8a"
+  },
+  {
+    "name": "system",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39.img.xz",
+    "hash": "cccd7073d067027396f2afd49874729757db0bbbc79853a0bf2938bd356fe164",
+    "hash_raw": "4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39",
+    "size": 5368709120,
+    "sparse": true,
+    "full_check": false,
+    "has_ab": true,
+    "ondevice_hash": "c7707f16ce7d977748677cc354e250943b4ff6c21b9a19a492053d32397cf9ec",
+    "alt": {
+      "hash": "4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-4bc3951f4aa3f70c53837dc2542d8b0666d37103b353fd81417cc7de1bbebe39.img",
+      "size": 5368709120
+    }
+  },
+  {
+    "name": "userdata_90",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_90-f0c675e0fae420870c9ba8979fa246b170f4f1a7a04b49609b55b6bdfa8c1b21.img.xz",
+    "hash": "3d8a007bae088c5959eb9b82454013f91868946d78380fecea2b1afdfb575c02",
+    "hash_raw": "f0c675e0fae420870c9ba8979fa246b170f4f1a7a04b49609b55b6bdfa8c1b21",
+    "size": 96636764160,
+    "sparse": true,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "5bfbabb8ff96b149056aa75d5b7e66a7cdd9cb4bcefe23b922c292f7f3a43462"
+  },
+  {
+    "name": "userdata_89",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_89-06fc52be37b42690ed7b4f8c66c4611309a2dea9fca37dd9d27d1eff302eb1bf.img.xz",
+    "hash": "443f136484294b210318842d09fb618d5411c8bdbab9f7421d8c89eb291a8d3f",
+    "hash_raw": "06fc52be37b42690ed7b4f8c66c4611309a2dea9fca37dd9d27d1eff302eb1bf",
+    "size": 95563022336,
+    "sparse": true,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "67db02b29a7e4435951c64cc962a474d048ed444aa912f3494391417cd51a074"
+  },
+  {
+    "name": "userdata_30",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_30-06679488f0c5c3fcfd5f351133050751cd189f705e478a979c45fc4a166d18a6.img.xz",
+    "hash": "875b580cb786f290a842e9187fd945657561886123eb3075a26f7995a18068f6",
+    "hash_raw": "06679488f0c5c3fcfd5f351133050751cd189f705e478a979c45fc4a166d18a6",
+    "size": 32212254720,
+    "sparse": true,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "16e27ba3c5cf9f0394ce6235ba6021b8a2de293fdb08399f8ca832fa5e4d0b9d"
+  }
+]

system/hardware/tici/amplifier.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+import time
+from smbus2 import SMBus
+from collections import namedtuple
+
+# https://datasheets.maximintegrated.com/en/ds/MAX98089.pdf
+
+AmpConfig = namedtuple('AmpConfig', ['name', 'value', 'register', 'offset', 'mask'])
+EQParams = namedtuple('EQParams', ['K', 'k1', 'k2', 'c1', 'c2'])
+
+def configs_from_eq_params(base, eq_params):
+  return [
+    AmpConfig("K (high)", (eq_params.K >> 8), base, 0, 0xFF),
+    AmpConfig("K (low)", (eq_params.K & 0xFF), base + 1, 0, 0xFF),
+    AmpConfig("k1 (high)", (eq_params.k1 >> 8), base + 2, 0, 0xFF),
+    AmpConfig("k1 (low)", (eq_params.k1 & 0xFF), base + 3, 0, 0xFF),
+    AmpConfig("k2 (high)", (eq_params.k2 >> 8), base + 4, 0, 0xFF),
+    AmpConfig("k2 (low)", (eq_params.k2 & 0xFF), base + 5, 0, 0xFF),
+    AmpConfig("c1 (high)", (eq_params.c1 >> 8), base + 6, 0, 0xFF),
+    AmpConfig("c1 (low)", (eq_params.c1 & 0xFF), base + 7, 0, 0xFF),
+    AmpConfig("c2 (high)", (eq_params.c2 >> 8), base + 8, 0, 0xFF),
+    AmpConfig("c2 (low)", (eq_params.c2 & 0xFF), base + 9, 0, 0xFF),
+  ]
+
+BASE_CONFIG = [
+  AmpConfig("MCLK prescaler", 0b01, 0x10, 4, 0b00110000),
+  AmpConfig("PM: enable speakers", 0b11, 0x4D, 4, 0b00110000),
+  AmpConfig("PM: enable DACs", 0b11, 0x4D, 0, 0b00000011),
+  AmpConfig("Enable PLL1", 0b1, 0x12, 7, 0b10000000),
+  AmpConfig("Enable PLL2", 0b1, 0x1A, 7, 0b10000000),
+  AmpConfig("DAI1: I2S mode", 0b00100, 0x14, 2, 0b01111100),
+  AmpConfig("DAI2: I2S mode", 0b00100, 0x1C, 2, 0b01111100),
+  AmpConfig("DAI1 Passband filtering: music mode", 0b1, 0x18, 7, 0b10000000),
+  AmpConfig("DAI1 voice mode gain (DV1G)", 0b00, 0x2F, 4, 0b00110000),
+  AmpConfig("DAI1 attenuation (DV1)", 0x0, 0x2F, 0, 0b00001111),
+  AmpConfig("DAI2 attenuation (DV2)", 0x0, 0x31, 0, 0b00001111),
+  AmpConfig("DAI2: DC blocking", 0b1, 0x20, 0, 0b00000001),
+  AmpConfig("DAI2: High sample rate", 0b0, 0x20, 3, 0b00001000),
+  AmpConfig("ALC enable", 0b1, 0x43, 7, 0b10000000),
+  AmpConfig("ALC/excursion limiter release time", 0b101, 0x43, 4, 0b01110000),
+  AmpConfig("ALC multiband enable", 0b1, 0x43, 3, 0b00001000),
+  AmpConfig("DAI1 EQ enable", 0b0, 0x49, 0, 0b00000001),
+  AmpConfig("DAI2 EQ clip detection disabled", 0b1, 0x32, 4, 0b00010000),
+  AmpConfig("DAI2 EQ attenuation", 0x5, 0x32, 0, 0b00001111),
+  AmpConfig("Excursion limiter upper corner freq", 0b100, 0x41, 4, 0b01110000),
+  AmpConfig("Excursion limiter lower corner freq", 0b00, 0x41, 0, 0b00000011),
+  AmpConfig("Excursion limiter threshold", 0b000, 0x42, 0, 0b00001111),
+  AmpConfig("Distortion limit (THDCLP)", 0x6, 0x46, 4, 0b11110000),
+  AmpConfig("Distortion limiter release time constant", 0b0, 0x46, 0, 0b00000001),
+  AmpConfig("Right DAC input mixer: DAI1 left", 0b0, 0x22, 3, 0b00001000),
+  AmpConfig("Right DAC input mixer: DAI1 right", 0b0, 0x22, 2, 0b00000100),
+  AmpConfig("Right DAC input mixer: DAI2 left", 0b1, 0x22, 1, 0b00000010),
+  AmpConfig("Right DAC input mixer: DAI2 right", 0b0, 0x22, 0, 0b00000001),
+  AmpConfig("DAI1 audio port selector", 0b10, 0x16, 6, 0b11000000),
+  AmpConfig("DAI2 audio port selector", 0b01, 0x1E, 6, 0b11000000),
+  AmpConfig("Enable left digital microphone", 0b1, 0x48, 5, 0b00100000),
+  AmpConfig("Enable right digital microphone", 0b1, 0x48, 4, 0b00010000),
+  AmpConfig("Enhanced volume smoothing disabled", 0b0, 0x49, 7, 0b10000000),
+  AmpConfig("Volume adjustment smoothing disabled", 0b0, 0x49, 6, 0b01000000),
+  AmpConfig("Zero-crossing detection disabled", 0b0, 0x49, 5, 0b00100000),
+]
+
+CONFIGS = {
+  "tici": [
+    AmpConfig("Right speaker output from right DAC", 0b1, 0x2C, 0, 0b11111111),
+    AmpConfig("Right Speaker Mixer Gain", 0b00, 0x2D, 2, 0b00001100),
+    AmpConfig("Right speaker output volume", 0x1c, 0x3E, 0, 0b00011111),
+    AmpConfig("DAI2 EQ enable", 0b1, 0x49, 1, 0b00000010),
+
+    *configs_from_eq_params(0x84, EQParams(0x274F, 0xC0FF, 0x3BF9, 0x0B3C, 0x1656)),
+    *configs_from_eq_params(0x8E, EQParams(0x1009, 0xC6BF, 0x2952, 0x1C97, 0x30DF)),
+    *configs_from_eq_params(0x98, EQParams(0x0F75, 0xCBE5, 0x0ED2, 0x2528, 0x3E42)),
+    *configs_from_eq_params(0xA2, EQParams(0x091F, 0x3D4C, 0xCE11, 0x1266, 0x2807)),
+    *configs_from_eq_params(0xAC, EQParams(0x0A9E, 0x3F20, 0xE573, 0x0A8B, 0x3A3B)),
+  ],
+  "tizi": [
+    AmpConfig("Left speaker output from left DAC", 0b1, 0x2B, 0, 0b11111111),
+    AmpConfig("Right speaker output from right DAC", 0b1, 0x2C, 0, 0b11111111),
+    AmpConfig("Left Speaker Mixer Gain", 0b00, 0x2D, 0, 0b00000011),
+    AmpConfig("Right Speaker Mixer Gain", 0b00, 0x2D, 2, 0b00001100),
+    AmpConfig("Left speaker output volume", 0x17, 0x3D, 0, 0b00011111),
+    AmpConfig("Right speaker output volume", 0x17, 0x3E, 0, 0b00011111),
+
+    AmpConfig("DAI2 EQ enable", 0b0, 0x49, 1, 0b00000010),
+    AmpConfig("DAI2: DC blocking", 0b0, 0x20, 0, 0b00000001),
+    AmpConfig("ALC enable", 0b0, 0x43, 7, 0b10000000),
+    AmpConfig("DAI2 EQ attenuation", 0x2, 0x32, 0, 0b00001111),
+    AmpConfig("Excursion limiter upper corner freq", 0b001, 0x41, 4, 0b01110000),
+    AmpConfig("Excursion limiter threshold", 0b100, 0x42, 0, 0b00001111),
+    AmpConfig("Distortion limit (THDCLP)", 0x0, 0x46, 4, 0b11110000),
+    AmpConfig("Distortion limiter release time constant", 0b1, 0x46, 0, 0b00000001),
+    AmpConfig("Left DAC input mixer: DAI1 left", 0b0, 0x22, 7, 0b10000000),
+    AmpConfig("Left DAC input mixer: DAI1 right", 0b0, 0x22, 6, 0b01000000),
+    AmpConfig("Left DAC input mixer: DAI2 left", 0b1, 0x22, 5, 0b00100000),
+    AmpConfig("Left DAC input mixer: DAI2 right", 0b0, 0x22, 4, 0b00010000),
+    AmpConfig("Right DAC input mixer: DAI2 left", 0b0, 0x22, 1, 0b00000010),
+    AmpConfig("Right DAC input mixer: DAI2 right", 0b1, 0x22, 0, 0b00000001),
+    AmpConfig("Volume adjustment smoothing disabled", 0b1, 0x49, 6, 0b01000000),
+  ],
+}
+
+class Amplifier:
+  AMP_I2C_BUS = 0
+  AMP_ADDRESS = 0x10
+
+  def __init__(self, debug=False):
+    self.debug = debug
+
+  def _get_shutdown_config(self, amp_disabled: bool) -> AmpConfig:
+    return AmpConfig("Global shutdown", 0b0 if amp_disabled else 0b1, 0x51, 7, 0b10000000)
+
+  def _set_configs(self, configs: list[AmpConfig]) -> None:
+    with SMBus(self.AMP_I2C_BUS) as bus:
+      for config in configs:
+        if self.debug:
+          print(f"Setting \"{config.name}\" to {config.value}:")
+
+        old_value = bus.read_byte_data(self.AMP_ADDRESS, config.register, force=True)
+        new_value = (old_value & (~config.mask)) | ((config.value << config.offset) & config.mask)
+        bus.write_byte_data(self.AMP_ADDRESS, config.register, new_value, force=True)
+
+        if self.debug:
+          print(f"  Changed {hex(config.register)}: {hex(old_value)} -> {hex(new_value)}")
+
+  def set_configs(self, configs: list[AmpConfig]) -> bool:
+    # retry in case panda is using the amp
+    tries = 15
+    backoff = 0.
+    for i in range(tries):
+      try:
+        self._set_configs(configs)
+        return True
+      except OSError:
+        backoff += 0.1
+        time.sleep(backoff)
+        print(f"Failed to set amp config, {tries - i - 1} retries left")
+    return False
+
+  def set_global_shutdown(self, amp_disabled: bool) -> bool:
+    return self.set_configs([self._get_shutdown_config(amp_disabled), ])
+
+  def initialize_configuration(self, model: str) -> bool:
+    cfgs = [
+      self._get_shutdown_config(True),
+      *BASE_CONFIG,
+      *CONFIGS[model],
+      self._get_shutdown_config(False),
+    ]
+    return self.set_configs(cfgs)
+
+
+if __name__ == "__main__":
+  with open("/sys/firmware/devicetree/base/model") as f:
+    model = f.read().strip('\x00')
+  model = model.split('comma ')[-1]
+
+  amp = Amplifier()
+  amp.initialize_configuration(model)

system/hardware/tici/esim.nmconnection

@@ -0,0 +1,30 @@
+[connection]
+id=esim
+uuid=fff6553c-3284-4707-a6b1-acc021caaafb
+type=gsm
+permissions=
+autoconnect=true
+autoconnect-retries=100
+autoconnect-priority=2
+metered=1
+
+[gsm]
+apn=
+home-only=false
+auto-config=true
+sim-id=
+
+[ipv4]
+route-metric=1000
+dns-priority=1000
+dns-search=
+method=auto
+
+[ipv6]
+ddr-gen-mode=stable-privacy
+dns-search=
+route-metric=1000
+dns-priority=1000
+method=auto
+
+[proxy]

system/hardware/tici/esim.py

@@ -0,0 +1,106 @@
+import json
+import os
+import shutil
+import subprocess
+from typing import Literal
+
+from openpilot.system.hardware.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+
+class TiciLPA(LPABase):
+  def __init__(self, interface: Literal['qmi', 'at'] = 'qmi'):
+    self.env = os.environ.copy()
+    self.env['LPAC_APDU'] = interface
+    self.env['QMI_DEVICE'] = '/dev/cdc-wdm0'
+    self.env['AT_DEVICE'] = '/dev/ttyUSB2'
+
+    self.timeout_sec = 45
+
+    if shutil.which('lpac') is None:
+      raise LPAError('lpac not found, must be installed!')
+
+  def list_profiles(self) -> list[Profile]:
+    msgs = self._invoke('profile', 'list')
+    self._validate_successful(msgs)
+    return [Profile(
+      iccid=p['iccid'],
+      nickname=p['profileNickname'],
+      enabled=p['profileState'] == 'enabled',
+      provider=p['serviceProviderName']
+    ) for p in msgs[-1]['payload']['data']]
+
+  def get_active_profile(self) -> Profile | None:
+    return next((p for p in self.list_profiles() if p.enabled), None)
+
+  def delete_profile(self, iccid: str) -> None:
+    self._validate_profile_exists(iccid)
+    latest = self.get_active_profile()
+    if latest is not None and latest.iccid == iccid:
+      raise LPAError('cannot delete active profile, switch to another profile first')
+    self._validate_successful(self._invoke('profile', 'delete', iccid))
+    self._process_notifications()
+
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    msgs = self._invoke('profile', 'download', '-a', qr)
+    self._validate_successful(msgs)
+    new_profile = next((m for m in msgs if m['payload']['message'] == 'es8p_meatadata_parse'), None)
+    if new_profile is None:
+      raise LPAError('no new profile found')
+    if nickname:
+      self.nickname_profile(new_profile['payload']['data']['iccid'], nickname)
+    self._process_notifications()
+
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    self._validate_profile_exists(iccid)
+    self._validate_successful(self._invoke('profile', 'nickname', iccid, nickname))
+
+  def switch_profile(self, iccid: str) -> None:
+    self._validate_profile_exists(iccid)
+    latest = self.get_active_profile()
+    if latest and latest.iccid == iccid:
+      return
+    self._validate_successful(self._invoke('profile', 'enable', iccid))
+    self._process_notifications()
+
+  def _invoke(self, *cmd: str):
+    proc = subprocess.Popen(['sudo', '-E', 'lpac'] + list(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.env)
+    try:
+      out, err = proc.communicate(timeout=self.timeout_sec)
+    except subprocess.TimeoutExpired as e:
+      proc.kill()
+      raise LPAError(f"lpac {cmd} timed out after {self.timeout_sec} seconds") from e
+
+    messages = []
+    for line in out.decode().strip().splitlines():
+      if line.startswith('{'):
+        message = json.loads(line)
+
+        # lpac response format validations
+        assert 'type' in message, 'expected type in message'
+        assert message['type'] == 'lpa' or message['type'] == 'progress', 'expected lpa or progress message type'
+        assert 'payload' in message, 'expected payload in message'
+        assert 'code' in message['payload'], 'expected code in message payload'
+        assert 'data' in message['payload'], 'expected data in message payload'
+
+        msg_ret_code = message['payload']['code']
+        if msg_ret_code != 0:
+          raise LPAError(f"lpac {' '.join(cmd)} failed with code {msg_ret_code}: <{message['payload']['message']}> {message['payload']['data']}")
+
+        messages.append(message)
+
+    if len(messages) == 0:
+      raise LPAError(f"lpac {cmd} returned no messages")
+
+    return messages
+
+  def _process_notifications(self) -> None:
+    """
+    Process notifications stored on the eUICC, typically to activate/deactivate the profile with the carrier.
+    """
+    self._validate_successful(self._invoke('notification', 'process', '-a', '-r'))
+
+  def _validate_profile_exists(self, iccid: str) -> None:
+    if not any(p.iccid == iccid for p in self.list_profiles()):
+      raise LPAProfileNotFoundError(f'profile {iccid} does not exist')
+
+  def _validate_successful(self, msgs: list[dict]) -> None:
+    assert msgs[-1]['payload']['message'] == 'success', 'expected success notification'

system/hardware/tici/hardware.h

@@ -0,0 +1,119 @@
+#pragma once
+
+#include <cstdlib>
+#include <cassert>
+#include <fstream>
+#include <map>
+#include <string>
+#include <algorithm>  // for std::clamp
+
+#include "common/params.h"
+#include "common/util.h"
+#include "system/hardware/base.h"
+
+class HardwareTici : public HardwareNone {
+public:
+  static constexpr float MAX_VOLUME = 0.9;
+  static constexpr float MIN_VOLUME = 0.1;
+  static bool TICI() { return true; }
+  static bool AGNOS() { return true; }
+  static std::string get_os_version() {
+    return "AGNOS " + util::read_file("/VERSION");
+  }
+
+  static std::string get_name() {
+    std::string model = util::read_file("/sys/firmware/devicetree/base/model");
+    return util::strip(model.substr(std::string("comma ").size()));
+  }
+
+  static cereal::InitData::DeviceType get_device_type() {
+    static const std::map<std::string, cereal::InitData::DeviceType> device_map = {
+      {"tici", cereal::InitData::DeviceType::TICI},
+      {"tizi", cereal::InitData::DeviceType::TIZI},
+      {"mici", cereal::InitData::DeviceType::MICI}
+    };
+    auto it = device_map.find(get_name());
+    assert(it != device_map.end());
+    return it->second;
+  }
+
+  static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); }
+  static int get_current() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/curr1_input").c_str()); }
+
+  static std::string get_serial() {
+    static std::string serial("");
+    if (serial.empty()) {
+      std::ifstream stream("/proc/cmdline");
+      std::string cmdline;
+      std::getline(stream, cmdline);
+
+      auto start = cmdline.find("serialno=");
+      if (start == std::string::npos) {
+        serial = "cccccc";
+      } else {
+        auto end = cmdline.find(" ", start + 9);
+        serial = cmdline.substr(start + 9, end - start - 9);
+      }
+    }
+    return serial;
+  }
+
+  static void reboot() { std::system("sudo reboot"); }
+  static void poweroff() { std::system("sudo poweroff"); }
+  static void set_brightness(int percent) {
+    float max = std::stof(util::read_file("/sys/class/backlight/panel0-backlight/max_brightness"));
+    std::ofstream("/sys/class/backlight/panel0-backlight/brightness") << int(percent * (max / 100.0f)) << "\n";
+  }
+  static void set_display_power(bool on) {
+    std::ofstream("/sys/class/backlight/panel0-backlight/bl_power") << (on ? "0" : "4") << "\n";
+  }
+
+  static void set_ir_power(int percent) {
+    auto device = get_device_type();
+    if (device == cereal::InitData::DeviceType::TICI ||
+        device == cereal::InitData::DeviceType::TIZI) {
+      return;
+    }
+
+    int value = util::map_val(std::clamp(percent, 0, 100), 0, 100, 0, 255);
+    std::ofstream("/sys/class/leds/led:switch_2/brightness") << 0 << "\n";
+    std::ofstream("/sys/class/leds/led:torch_2/brightness") << value << "\n";
+    std::ofstream("/sys/class/leds/led:switch_2/brightness") << value << "\n";
+  }
+
+  static std::map<std::string, std::string> get_init_logs() {
+    std::map<std::string, std::string> ret = {
+      {"/BUILD", util::read_file("/BUILD")},
+      {"lsblk", util::check_output("lsblk -o NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL")},
+      {"SOM ID", util::read_file("/sys/devices/platform/vendor/vendor:gpio-som-id/som_id")},
+    };
+
+    std::string bs = util::check_output("abctl --boot_slot");
+    ret["boot slot"] = bs.substr(0, bs.find_first_of("\n"));
+
+    std::string temp = util::read_file("/dev/disk/by-partlabel/ssd");
+    temp.erase(temp.find_last_not_of(std::string("\0\r\n", 3))+1);
+    ret["boot temp"] = temp;
+
+    // TODO: log something from system and boot
+    for (std::string part : {"xbl", "abl", "aop", "devcfg", "xbl_config"}) {
+      for (std::string slot : {"a", "b"}) {
+        std::string partition = part + "_" + slot;
+        std::string hash = util::check_output("sha256sum /dev/disk/by-partlabel/" + partition);
+        ret[partition] = hash.substr(0, hash.find_first_of(" "));
+      }
+    }
+
+    return ret;
+  }
+
+  static bool get_ssh_enabled() { return Params().getBool("SshEnabled"); }
+  static void set_ssh_enabled(bool enabled) { Params().putBool("SshEnabled", enabled); }
+
+  static void config_cpu_rendering(bool offscreen) {
+    if (offscreen) {
+      setenv("QT_QPA_PLATFORM", "eglfs", 1); // offscreen doesn't work with EGL/GLES
+    }
+    setenv("LP_NUM_THREADS", "0", 1); // disable threading so we stay on our assigned CPU
+  }
+};

system/hardware/tici/hardware.py

@@ -0,0 +1,594 @@
+import json
+import math
+import os
+import subprocess
+import time
+import tempfile
+from enum import IntEnum
+from functools import cached_property, lru_cache
+from pathlib import Path
+
+from cereal import log
+from openpilot.common.util import sudo_read, sudo_write
+from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
+from openpilot.system.hardware.base import HardwareBase, LPABase, ThermalConfig, ThermalZone
+from openpilot.system.hardware.tici import iwlist
+from openpilot.system.hardware.tici.esim import TiciLPA
+from openpilot.system.hardware.tici.pins import GPIO
+from openpilot.system.hardware.tici.amplifier import Amplifier
+
+NM = 'org.freedesktop.NetworkManager'
+NM_CON_ACT = NM + '.Connection.Active'
+NM_DEV = NM + '.Device'
+NM_DEV_WL = NM + '.Device.Wireless'
+NM_DEV_STATS = NM + '.Device.Statistics'
+NM_AP = NM + '.AccessPoint'
+DBUS_PROPS = 'org.freedesktop.DBus.Properties'
+
+MM = 'org.freedesktop.ModemManager1'
+MM_MODEM = MM + ".Modem"
+MM_MODEM_SIMPLE = MM + ".Modem.Simple"
+MM_SIM = MM + ".Sim"
+
+class MM_MODEM_STATE(IntEnum):
+  FAILED        = -1
+  UNKNOWN       = 0
+  INITIALIZING  = 1
+  LOCKED        = 2
+  DISABLED      = 3
+  DISABLING     = 4
+  ENABLING      = 5
+  ENABLED       = 6
+  SEARCHING     = 7
+  REGISTERED    = 8
+  DISCONNECTING = 9
+  CONNECTING    = 10
+  CONNECTED     = 11
+
+class NMMetered(IntEnum):
+  NM_METERED_UNKNOWN = 0
+  NM_METERED_YES = 1
+  NM_METERED_NO = 2
+  NM_METERED_GUESS_YES = 3
+  NM_METERED_GUESS_NO = 4
+
+TIMEOUT = 0.1
+REFRESH_RATE_MS = 1000
+
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+
+# https://developer.gnome.org/ModemManager/unstable/ModemManager-Flags-and-Enumerations.html#MMModemAccessTechnology
+MM_MODEM_ACCESS_TECHNOLOGY_UMTS = 1 << 5
+MM_MODEM_ACCESS_TECHNOLOGY_LTE = 1 << 14
+
+
+def affine_irq(val, action):
+  irqs = get_irqs_for_action(action)
+  if len(irqs) == 0:
+    print(f"No IRQs found for '{action}'")
+    return
+
+  for i in irqs:
+    sudo_write(str(val), f"/proc/irq/{i}/smp_affinity_list")
+
+@lru_cache
+def get_device_type():
+  # lru_cache and cache can cause memory leaks when used in classes
+  with open("/sys/firmware/devicetree/base/model") as f:
+    model = f.read().strip('\x00')
+  return model.split('comma ')[-1]
+
+class Tici(HardwareBase):
+  @cached_property
+  def bus(self):
+    import dbus
+    return dbus.SystemBus()
+
+  @cached_property
+  def nm(self):
+    return self.bus.get_object(NM, '/org/freedesktop/NetworkManager')
+
+  @property # this should not be cached, in case the modemmanager restarts
+  def mm(self):
+    return self.bus.get_object(MM, '/org/freedesktop/ModemManager1')
+
+  @cached_property
+  def amplifier(self):
+    if self.get_device_type() == "mici":
+      return None
+    return Amplifier()
+
+  def get_os_version(self):
+    with open("/VERSION") as f:
+      return f.read().strip()
+
+  def get_device_type(self):
+    return get_device_type()
+
+  def reboot(self, reason=None):
+    subprocess.check_output(["sudo", "reboot"])
+
+  def uninstall(self):
+    Path("/data/__system_reset__").touch()
+    os.sync()
+    self.reboot()
+
+  def get_serial(self):
+    return self.get_cmdline()['androidboot.serialno']
+
+  def get_network_type(self):
+    try:
+      primary_connection = self.nm.Get(NM, 'PrimaryConnection', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+      primary_connection = self.bus.get_object(NM, primary_connection)
+      primary_type = primary_connection.Get(NM_CON_ACT, 'Type', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+
+      if primary_type == '802-3-ethernet':
+        return NetworkType.ethernet
+      elif primary_type == '802-11-wireless':
+        return NetworkType.wifi
+      else:
+        active_connections = self.nm.Get(NM, 'ActiveConnections', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+        for conn in active_connections:
+          c = self.bus.get_object(NM, conn)
+          tp = c.Get(NM_CON_ACT, 'Type', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+          if tp == 'gsm':
+            modem = self.get_modem()
+            access_t = modem.Get(MM_MODEM, 'AccessTechnologies', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+            if access_t >= MM_MODEM_ACCESS_TECHNOLOGY_LTE:
+              return NetworkType.cell4G
+            elif access_t >= MM_MODEM_ACCESS_TECHNOLOGY_UMTS:
+              return NetworkType.cell3G
+            else:
+              return NetworkType.cell2G
+    except Exception:
+      pass
+
+    return NetworkType.none
+
+  def get_modem(self):
+    objects = self.mm.GetManagedObjects(dbus_interface="org.freedesktop.DBus.ObjectManager", timeout=TIMEOUT)
+    modem_path = list(objects.keys())[0]
+    return self.bus.get_object(MM, modem_path)
+
+  def get_wlan(self):
+    wlan_path = self.nm.GetDeviceByIpIface('wlan0', dbus_interface=NM, timeout=TIMEOUT)
+    return self.bus.get_object(NM, wlan_path)
+
+  def get_wwan(self):
+    wwan_path = self.nm.GetDeviceByIpIface('wwan0', dbus_interface=NM, timeout=TIMEOUT)
+    return self.bus.get_object(NM, wwan_path)
+
+  def get_sim_info(self):
+    modem = self.get_modem()
+    sim_path = modem.Get(MM_MODEM, 'Sim', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+
+    if sim_path == "/":
+      return {
+        'sim_id': '',
+        'mcc_mnc': None,
+        'network_type': ["Unknown"],
+        'sim_state': ["ABSENT"],
+        'data_connected': False
+      }
+    else:
+      sim = self.bus.get_object(MM, sim_path)
+      return {
+        'sim_id': str(sim.Get(MM_SIM, 'SimIdentifier', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)),
+        'mcc_mnc': str(sim.Get(MM_SIM, 'OperatorIdentifier', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)),
+        'network_type': ["Unknown"],
+        'sim_state': ["READY"],
+        'data_connected': modem.Get(MM_MODEM, 'State', dbus_interface=DBUS_PROPS, timeout=TIMEOUT) == MM_MODEM_STATE.CONNECTED,
+      }
+
+  def get_sim_lpa(self) -> LPABase:
+    return TiciLPA()
+
+  def get_imei(self, slot):
+    if slot != 0:
+      return ""
+
+    return str(self.get_modem().Get(MM_MODEM, 'EquipmentIdentifier', dbus_interface=DBUS_PROPS, timeout=TIMEOUT))
+
+  def get_network_info(self):
+    if self.get_device_type() == "mici":
+      return None
+    try:
+      modem = self.get_modem()
+      info = modem.Command("AT+QNWINFO", math.ceil(TIMEOUT), dbus_interface=MM_MODEM, timeout=TIMEOUT)
+      extra = modem.Command('AT+QENG="servingcell"', math.ceil(TIMEOUT), dbus_interface=MM_MODEM, timeout=TIMEOUT)
+      state = modem.Get(MM_MODEM, 'State', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+    except Exception:
+      return None
+
+    if info and info.startswith('+QNWINFO: '):
+      info = info.replace('+QNWINFO: ', '').replace('"', '').split(',')
+      extra = "" if extra is None else extra.replace('+QENG: "servingcell",', '').replace('"', '')
+      state = "" if state is None else MM_MODEM_STATE(state).name
+
+      if len(info) != 4:
+        return None
+
+      technology, operator, band, channel = info
+
+      return({
+        'technology': technology,
+        'operator': operator,
+        'band': band,
+        'channel': int(channel),
+        'extra': extra,
+        'state': state,
+      })
+    else:
+      return None
+
+  def parse_strength(self, percentage):
+    if percentage < 25:
+      return NetworkStrength.poor
+    elif percentage < 50:
+      return NetworkStrength.moderate
+    elif percentage < 75:
+      return NetworkStrength.good
+    else:
+      return NetworkStrength.great
+
+  def get_network_strength(self, network_type):
+    network_strength = NetworkStrength.unknown
+
+    try:
+      if network_type == NetworkType.none:
+        pass
+      elif network_type == NetworkType.wifi:
+        wlan = self.get_wlan()
+        active_ap_path = wlan.Get(NM_DEV_WL, 'ActiveAccessPoint', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+        if active_ap_path != "/":
+          active_ap = self.bus.get_object(NM, active_ap_path)
+          strength = int(active_ap.Get(NM_AP, 'Strength', dbus_interface=DBUS_PROPS, timeout=TIMEOUT))
+          network_strength = self.parse_strength(strength)
+      else:  # Cellular
+        modem = self.get_modem()
+        strength = int(modem.Get(MM_MODEM, 'SignalQuality', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)[0])
+        network_strength = self.parse_strength(strength)
+    except Exception:
+      pass
+
+    return network_strength
+
+  def get_network_metered(self, network_type) -> bool:
+    try:
+      primary_connection = self.nm.Get(NM, 'PrimaryConnection', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+      primary_connection = self.bus.get_object(NM, primary_connection)
+      primary_devices = primary_connection.Get(NM_CON_ACT, 'Devices', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+
+      for dev in primary_devices:
+        dev_obj = self.bus.get_object(NM, str(dev))
+        metered_prop = dev_obj.Get(NM_DEV, 'Metered', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+
+        if network_type == NetworkType.wifi:
+          if metered_prop in [NMMetered.NM_METERED_YES, NMMetered.NM_METERED_GUESS_YES]:
+            return True
+        elif network_type in [NetworkType.cell2G, NetworkType.cell3G, NetworkType.cell4G, NetworkType.cell5G]:
+          if metered_prop == NMMetered.NM_METERED_NO:
+            return False
+    except Exception:
+      pass
+
+    return super().get_network_metered(network_type)
+
+  def get_modem_version(self):
+    try:
+      modem = self.get_modem()
+      return modem.Get(MM_MODEM, 'Revision', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+    except Exception:
+      return None
+
+  def get_modem_temperatures(self):
+    timeout = 0.2  # Default timeout is too short
+    try:
+      modem = self.get_modem()
+      temps = modem.Command("AT+QTEMP", math.ceil(timeout), dbus_interface=MM_MODEM, timeout=timeout)
+      return list(filter(lambda t: t != 255, map(int, temps.split(' ')[1].split(','))))
+    except Exception:
+      return []
+
+  def get_nvme_temperatures(self):
+    ret = []
+    try:
+      out = subprocess.check_output("sudo smartctl -aj /dev/nvme0", shell=True)
+      dat = json.loads(out)
+      ret = list(map(int, dat["nvme_smart_health_information_log"]["temperature_sensors"]))
+    except Exception:
+      pass
+    return ret
+
+  def get_current_power_draw(self):
+    return (self.read_param_file("/sys/class/hwmon/hwmon1/power1_input", int) / 1e6)
+
+  def get_som_power_draw(self):
+    return (self.read_param_file("/sys/class/power_supply/bms/voltage_now", int) * self.read_param_file("/sys/class/power_supply/bms/current_now", int) / 1e12)
+
+  def shutdown(self):
+    os.system("sudo poweroff")
+
+  def get_thermal_config(self):
+    intake, exhaust, case = None, None, None
+    if self.get_device_type() == "mici":
+      case = ThermalZone("case")
+      intake = ThermalZone("intake")
+      exhaust = ThermalZone("exhaust")
+    return ThermalConfig(cpu=[ThermalZone(f"cpu{i}-silver-usr") for i in range(4)] +
+                             [ThermalZone(f"cpu{i}-gold-usr") for i in range(4)],
+                         gpu=[ThermalZone("gpu0-usr"), ThermalZone("gpu1-usr")],
+                         dsp=ThermalZone("compute-hvx-usr"),
+                         memory=ThermalZone("ddr-usr"),
+                         pmic=[ThermalZone("pm8998_tz"), ThermalZone("pm8005_tz")],
+                         intake=intake,
+                         exhaust=exhaust,
+                         case=case)
+
+  def set_display_power(self, on):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/bl_power", "w") as f:
+        f.write("0" if on else "4")
+    except Exception:
+      pass
+
+  def set_screen_brightness(self, percentage):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
+        max_brightness = float(f.read().strip())
+
+      val = int(percentage * (max_brightness / 100.))
+      with open("/sys/class/backlight/panel0-backlight/brightness", "w") as f:
+        f.write(str(val))
+    except Exception:
+      pass
+
+  def get_screen_brightness(self):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
+        max_brightness = float(f.read().strip())
+
+      with open("/sys/class/backlight/panel0-backlight/brightness") as f:
+        return int(float(f.read()) / (max_brightness / 100.))
+    except Exception:
+      return 0
+
+  def set_power_save(self, powersave_enabled):
+    # amplifier, 100mW at idle
+    if self.amplifier is not None:
+      self.amplifier.set_global_shutdown(amp_disabled=powersave_enabled)
+      if not powersave_enabled:
+        self.amplifier.initialize_configuration(self.get_device_type())
+
+    # *** CPU config ***
+
+    # offline big cluster
+    for i in range(4, 8):
+      val = '0' if powersave_enabled else '1'
+      sudo_write(val, f'/sys/devices/system/cpu/cpu{i}/online')
+
+    for n in ('0', '4'):
+      if powersave_enabled and n == '4':
+        continue
+      gov = 'ondemand' if powersave_enabled else 'performance'
+      sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
+
+    # *** IRQ config ***
+
+    # GPU, modeld core
+    affine_irq(7, "kgsl-3d0")
+
+    # camerad core
+    camera_irqs = ("a5", "cci", "cpas_camnoc", "cpas-cdm", "csid", "ife", "csid-lite", "ife-lite")
+    for n in camera_irqs:
+      affine_irq(6, n)
+
+  def get_gpu_usage_percent(self):
+    try:
+      with open('/sys/class/kgsl/kgsl-3d0/gpubusy') as f:
+        used, total = f.read().strip().split()
+      return 100.0 * int(used) / int(total)
+    except Exception:
+      return 0
+
+  def initialize_hardware(self):
+    if self.amplifier is not None:
+      self.amplifier.initialize_configuration(self.get_device_type())
+
+    # Allow hardwared to write engagement status to kmsg
+    os.system("sudo chmod a+w /dev/kmsg")
+
+    # Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
+    gpio_init(GPIO.SOM_ST_IO, True)
+    gpio_set(GPIO.SOM_ST_IO, 1)
+
+    # *** IRQ config ***
+
+    # mask off big cluster from default affinity
+    sudo_write("f", "/proc/irq/default_smp_affinity")
+
+    # move these off the default core
+    affine_irq(1, "msm_drm")   # display
+    affine_irq(1, "msm_vidc")  # encoders
+    affine_irq(1, "i2c_geni")  # sensors
+
+    # *** GPU config ***
+    # https://github.com/commaai/agnos-kernel-sdm845/blob/master/arch/arm64/boot/dts/qcom/sdm845-gpu.dtsi#L216
+    sudo_write("0", "/sys/class/kgsl/kgsl-3d0/min_pwrlevel")
+    sudo_write("0", "/sys/class/kgsl/kgsl-3d0/max_pwrlevel")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_bus_on")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_clk_on")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_rail_on")
+    sudo_write("1000", "/sys/class/kgsl/kgsl-3d0/idle_timer")
+    sudo_write("performance", "/sys/class/kgsl/kgsl-3d0/devfreq/governor")
+    sudo_write("710", "/sys/class/kgsl/kgsl-3d0/max_clock_mhz")
+
+    # setup governors
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,cpubw/governor")
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu0/governor")
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu4/governor")
+
+    # *** VIDC (encoder) config ***
+    sudo_write("N", "/sys/kernel/debug/msm_vidc/clock_scaling")
+    sudo_write("Y", "/sys/kernel/debug/msm_vidc/disable_thermal_mitigation")
+
+    # pandad core
+    affine_irq(3, "spi_geni")         # SPI
+    if "tici" in self.get_device_type():
+      affine_irq(3, "xhci-hcd:usb3")  # aux panda USB (or potentially anything else on USB)
+      affine_irq(3, "xhci-hcd:usb1")  # internal panda USB (also modem)
+    try:
+      pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
+      subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
+      subprocess.call(["sudo", "taskset", "-pc", "3", pid])
+    except subprocess.CalledProcessException as e:
+      print(str(e))
+
+  def configure_modem(self):
+    sim_id = self.get_sim_info().get('sim_id', '')
+
+    modem = self.get_modem()
+    try:
+      manufacturer = str(modem.Get(MM_MODEM, 'Manufacturer', dbus_interface=DBUS_PROPS, timeout=TIMEOUT))
+    except Exception:
+      manufacturer = None
+
+    cmds = []
+
+    if self.get_device_type() in ("tici", "tizi"):
+      # clear out old blue prime initial APN
+      os.system('mmcli -m any --3gpp-set-initial-eps-bearer-settings="apn="')
+
+      cmds += [
+        # configure modem as data-centric
+        'AT+QNVW=5280,0,"0102000000000000"',
+        'AT+QNVFW="/nv/item_files/ims/IMS_enable",00',
+        'AT+QNVFW="/nv/item_files/modem/mmode/ue_usage_setting",01',
+      ]
+      if self.get_device_type() == "tizi":
+        # SIM hot swap, not routed on tici
+        cmds += [
+          'AT+QSIMDET=1,0',
+          'AT+QSIMSTAT=1',
+        ]
+    elif manufacturer == 'Cavli Inc.':
+      cmds += [
+        'AT^SIMSWAP=1',     # use SIM slot, instead of internal eSIM
+        'AT$QCSIMSLEEP=0',  # disable SIM sleep
+        'AT$QCSIMCFG=SimPowerSave,0',  # more sleep disable
+
+        # ethernet config
+        'AT$QCPCFG=usbNet,0',
+        'AT$QCNETDEVCTL=3,1',
+      ]
+    else:
+      # this modem gets upset with too many AT commands
+      if sim_id is None or len(sim_id) == 0:
+        cmds += [
+          # SIM sleep disable
+          'AT$QCSIMSLEEP=0',
+          'AT$QCSIMCFG=SimPowerSave,0',
+
+          # ethernet config
+          'AT$QCPCFG=usbNet,1',
+        ]
+
+    for cmd in cmds:
+      try:
+        modem.Command(cmd, math.ceil(TIMEOUT), dbus_interface=MM_MODEM, timeout=TIMEOUT)
+      except Exception:
+        pass
+
+    # eSIM prime
+    dest = "/etc/NetworkManager/system-connections/esim.nmconnection"
+    if sim_id.startswith('8985235') and not os.path.exists(dest):
+      with open(Path(__file__).parent/'esim.nmconnection') as f, tempfile.NamedTemporaryFile(mode='w') as tf:
+        dat = f.read()
+        dat = dat.replace("sim-id=", f"sim-id={sim_id}")
+        tf.write(dat)
+        tf.flush()
+
+        # needs to be root
+        os.system(f"sudo cp {tf.name} {dest}")
+      os.system(f"sudo nmcli con load {dest}")
+
+  def get_networks(self):
+    r = {}
+
+    wlan = iwlist.scan()
+    if wlan is not None:
+      r['wlan'] = wlan
+
+    lte_info = self.get_network_info()
+    if lte_info is not None:
+      extra = lte_info['extra']
+
+      # <state>,"LTE",<is_tdd>,<mcc>,<mnc>,<cellid>,<pcid>,<earfcn>,<freq_band_ind>,
+      # <ul_bandwidth>,<dl_bandwidth>,<tac>,<rsrp>,<rsrq>,<rssi>,<sinr>,<srxlev>
+      if 'LTE' in extra:
+        extra = extra.split(',')
+        try:
+          r['lte'] = [{
+            "mcc": int(extra[3]),
+            "mnc": int(extra[4]),
+            "cid": int(extra[5], 16),
+            "nmr": [{"pci": int(extra[6]), "earfcn": int(extra[7])}],
+          }]
+        except (ValueError, IndexError):
+          pass
+
+    return r
+
+  def get_modem_data_usage(self):
+    try:
+      wwan = self.get_wwan()
+
+      # Ensure refresh rate is set so values don't go stale
+      refresh_rate = wwan.Get(NM_DEV_STATS, 'RefreshRateMs', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+      if refresh_rate != REFRESH_RATE_MS:
+        u = type(refresh_rate)
+        wwan.Set(NM_DEV_STATS, 'RefreshRateMs', u(REFRESH_RATE_MS), dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+
+      tx = wwan.Get(NM_DEV_STATS, 'TxBytes', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+      rx = wwan.Get(NM_DEV_STATS, 'RxBytes', dbus_interface=DBUS_PROPS, timeout=TIMEOUT)
+      return int(tx), int(rx)
+    except Exception:
+      return -1, -1
+
+  def has_internal_panda(self):
+    return True
+
+  def reset_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_init(GPIO.STM_BOOT0, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    gpio_set(GPIO.STM_BOOT0, 0)
+    time.sleep(1)
+    gpio_set(GPIO.STM_RST_N, 0)
+
+  def recover_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_init(GPIO.STM_BOOT0, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    gpio_set(GPIO.STM_BOOT0, 1)
+    time.sleep(0.5)
+    gpio_set(GPIO.STM_RST_N, 0)
+    time.sleep(0.5)
+    gpio_set(GPIO.STM_BOOT0, 0)
+
+  def booted(self):
+    # this normally boots within 8s, but on rare occasions takes 30+s
+    encoder_state = sudo_read("/sys/kernel/debug/msm_vidc/core0/info")
+    if "Core state: 0" in encoder_state and (time.monotonic() < 60*2):
+      return False
+    return True
+
+if __name__ == "__main__":
+  t = Tici()
+  t.configure_modem()
+  t.initialize_hardware()
+  t.set_power_save(False)
+  print(t.get_sim_info())

system/hardware/tici/id_rsa

@@ -0,0 +1,28 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
+Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
+t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
+Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
+acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
+1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
+rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
+x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
+gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
+Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
+LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
+s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
+Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
+6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
+uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
+0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
+FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
+ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
+4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
+nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
+VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
+s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
+jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
+b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
+RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
+6co17RFaAZHwGfCFFjO76Q==
+-----END RSA PRIVATE KEY-----

system/hardware/tici/iwlist.py

@@ -0,0 +1,35 @@
+import subprocess
+
+
+def scan(interface="wlan0"):
+  result = []
+  try:
+    r = subprocess.check_output(["iwlist", interface, "scan"], encoding='utf8')
+
+    mac = None
+    for line in r.split('\n'):
+      if "Address" in line:
+        # Based on the adapter eithere a percentage or dBm is returned
+        # Add previous network in case no dBm signal level was seen
+        if mac is not None:
+          result.append({"mac": mac})
+          mac = None
+
+        mac = line.split(' ')[-1]
+      elif "dBm" in line:
+        try:
+          level = line.split('Signal level=')[1]
+          rss = int(level.split(' ')[0])
+          result.append({"mac": mac, "rss": rss})
+          mac = None
+        except ValueError:
+          continue
+
+    # Add last network if no dBm was found
+    if mac is not None:
+      result.append({"mac": mac})
+
+    return result
+
+  except Exception:
+    return None

system/hardware/tici/pins.py

@@ -0,0 +1,32 @@
+# TODO: these are also defined in a header
+
+# GPIO pin definitions
+class GPIO:
+  # both GPIO_STM_RST_N and GPIO_LTE_RST_N are misnamed, they are high to reset
+  HUB_RST_N = 30
+  UBLOX_RST_N = 32
+  UBLOX_SAFEBOOT_N = 33
+  GNSS_PWR_EN = 34 # SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN
+
+  STM_RST_N = 124
+  STM_BOOT0 = 134
+  STM_PWR_EN_N = 41 # because STM32H7 RST doesn't generate a full power-on-reset
+
+  SIREN = 42
+  SOM_ST_IO = 49
+
+  LTE_RST_N = 50
+  LTE_PWRKEY = 116
+  LTE_BOOT = 52
+
+  # GPIO_CAM0_DVDD_EN = /sys/kernel/debug/regulator/camera_rear_ldo
+  CAM0_AVDD_EN = 8
+  CAM0_RSTN = 9
+  CAM1_RSTN = 7
+  CAM2_RSTN = 12
+
+  # Sensor interrupts
+  BMX055_ACCEL_INT = 21
+  BMX055_GYRO_INT = 23
+  BMX055_MAGN_INT = 87
+  LSM_INT = 84

system/hardware/tici/power_monitor.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import sys
+import time
+import datetime
+import numpy as np
+from collections import deque
+
+from openpilot.common.realtime import Ratekeeper
+from openpilot.common.filter_simple import FirstOrderFilter
+
+
+def read_power():
+  with open("/sys/bus/i2c/devices/0-0040/hwmon/hwmon1/power1_input") as f:
+    return int(f.read()) / 1e6
+
+def sample_power(seconds=5) -> list[float]:
+  rate = 123
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+
+  pwrs = []
+  for _ in range(rate*seconds):
+    pwrs.append(read_power())
+    rk.keep_time()
+  return pwrs
+
+def get_power(seconds=5):
+  pwrs = sample_power(seconds)
+  return np.mean(pwrs)
+
+def wait_for_power(min_pwr, max_pwr, min_secs_in_range, timeout):
+  start_time = time.monotonic()
+  pwrs = deque([min_pwr - 1.]*min_secs_in_range, maxlen=min_secs_in_range)
+  while (time.monotonic() - start_time < timeout):
+    pwrs.append(get_power(1))
+    if all(min_pwr <= p <= max_pwr for p in pwrs):
+      break
+  return np.mean(pwrs)
+
+
+if __name__ == "__main__":
+  duration = None
+  if len(sys.argv) > 1:
+    duration = int(sys.argv[1])
+
+  rate = 23
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+  fltr = FirstOrderFilter(0, 5, 1. / rate, initialized=False)
+
+  measurements = []
+  start_time = time.monotonic()
+
+  try:
+    while duration is None or time.monotonic() - start_time < duration:
+      fltr.update(read_power())
+      if rk.frame % rate == 0:
+        measurements.append(fltr.x)
+        t = datetime.timedelta(seconds=time.monotonic() - start_time)
+        avg = sum(measurements) / len(measurements)
+        print(f"Now: {fltr.x:.2f} W, Avg: {avg:.2f} W over {t}")
+      rk.keep_time()
+  except KeyboardInterrupt:
+    pass
+
+  t = datetime.timedelta(seconds=time.monotonic() - start_time)
+  avg = sum(measurements) / len(measurements)
+  print(f"\nAverage power: {avg:.2f}W over {t}")

system/hardware/tici/precise_power_measure.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+import numpy as np
+from openpilot.system.hardware.tici.power_monitor import sample_power
+
+if __name__ == '__main__':
+  print("measuring for 5 seconds")
+  for _ in range(3):
+    pwrs = sample_power()
+    print(f"mean {np.mean(pwrs):.2f} std {np.std(pwrs):.2f}")

system/hardware/tici/restart_modem.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#nmcli connection modify --temporary lte gsm.home-only yes
+#nmcli connection modify --temporary lte gsm.auto-config yes
+#nmcli connection modify --temporary lte connection.autoconnect-retries 20
+sudo nmcli connection reload
+
+sudo systemctl stop ModemManager
+nmcli con down lte
+nmcli con down blue-prime
+
+# power cycle modem
+/usr/comma/lte/lte.sh stop_blocking
+/usr/comma/lte/lte.sh start
+
+sudo systemctl restart NetworkManager
+#sudo systemctl restart ModemManager
+sudo ModemManager --debug

system/hardware/tici/tests/compare_casync_manifest.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import argparse
+import collections
+import multiprocessing
+import os
+
+import requests
+from tqdm import tqdm
+
+import openpilot.system.hardware.tici.casync as casync
+
+
+def get_chunk_download_size(chunk):
+  sha = chunk.sha.hex()
+  path = os.path.join(remote_url, sha[:4], sha + ".cacnk")
+  if os.path.isfile(path):
+    return os.path.getsize(path)
+  else:
+    r = requests.head(path, timeout=10)
+    r.raise_for_status()
+    return int(r.headers['content-length'])
+
+
+if __name__ == "__main__":
+
+  parser = argparse.ArgumentParser(description='Compute overlap between two casync manifests')
+  parser.add_argument('frm')
+  parser.add_argument('to')
+  args = parser.parse_args()
+
+  frm = casync.parse_caibx(args.frm)
+  to = casync.parse_caibx(args.to)
+  remote_url = args.to.replace('.caibx', '')
+
+  most_common = collections.Counter(t.sha for t in to).most_common(1)[0][0]
+
+  frm_dict = casync.build_chunk_dict(frm)
+
+  # Get content-length for each chunk
+  with multiprocessing.Pool() as pool:
+    szs = list(tqdm(pool.imap(get_chunk_download_size, to), total=len(to)))
+  chunk_sizes = {t.sha: sz for (t, sz) in zip(to, szs, strict=True)}
+
+  sources: dict[str, list[int]] = {
+    'seed': [],
+    'remote_uncompressed': [],
+    'remote_compressed': [],
+  }
+
+  for chunk in to:
+    # Assume most common chunk is the zero chunk
+    if chunk.sha == most_common:
+      continue
+
+    if chunk.sha in frm_dict:
+      sources['seed'].append(chunk.length)
+    else:
+      sources['remote_uncompressed'].append(chunk.length)
+      sources['remote_compressed'].append(chunk_sizes[chunk.sha])
+
+  print()
+  print("Update statistics (excluding zeros)")
+  print()
+  print("Download only with no seed:")
+  print(f"  Remote (uncompressed)\t\t{sum(sources['seed'] + sources['remote_uncompressed']) / 1000 / 1000:.2f} MB\tn = {len(to)}")
+  print(f"  Remote (compressed download)\t{sum(chunk_sizes.values()) / 1000 / 1000:.2f} MB\tn = {len(to)}")
+  print()
+  print("Upgrade with seed partition:")
+  print(f"  Seed   (uncompressed)\t\t{sum(sources['seed']) / 1000 / 1000:.2f} MB\t\t\t\tn = {len(sources['seed'])}")
+  sz, n = sum(sources['remote_uncompressed']), len(sources['remote_uncompressed'])
+  print(f"  Remote (uncompressed)\t\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}")
+  sz, n = sum(sources['remote_compressed']), len(sources['remote_compressed'])
+  print(f"  Remote (compressed download)\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}")

system/hardware/tici/tests/test_agnos_updater.py

@@ -0,0 +1,20 @@
+import json
+import os
+import requests
+
+TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
+
+
+class TestAgnosUpdater:
+
+  def test_manifest(self):
+    with open(MANIFEST) as f:
+      m = json.load(f)
+
+    for img in m:
+      r = requests.head(img['url'], timeout=10)
+      r.raise_for_status()
+      assert r.headers['Content-Type'] == "application/x-xz"
+      if not img['sparse']:
+        assert img['hash'] == img['hash_raw']

system/hardware/tici/tests/test_amplifier.py

@@ -0,0 +1,70 @@
+import pytest
+import time
+import random
+import subprocess
+
+from panda import Panda
+from openpilot.system.hardware import TICI, HARDWARE
+from openpilot.system.hardware.tici.hardware import Tici
+from openpilot.system.hardware.tici.amplifier import Amplifier
+
+
+class TestAmplifier:
+
+  @classmethod
+  def setup_class(cls):
+    if not TICI:
+      pytest.skip()
+
+  def setup_method(self):
+    # clear dmesg
+    subprocess.check_call("sudo dmesg -C", shell=True)
+
+    HARDWARE.reset_internal_panda()
+    Panda.wait_for_panda(None, 30)
+    self.panda = Panda()
+
+  def teardown_method(self):
+    HARDWARE.reset_internal_panda()
+
+  def _check_for_i2c_errors(self, expected):
+    dmesg = subprocess.check_output("dmesg", shell=True, encoding='utf8')
+    i2c_lines = [l for l in dmesg.strip().splitlines() if 'i2c_geni a88000.i2c' in l]
+    i2c_str = '\n'.join(i2c_lines)
+
+    if not expected:
+      return len(i2c_lines) == 0
+    else:
+      return "i2c error :-107" in i2c_str or "Bus arbitration lost" in i2c_str
+
+  def test_init(self):
+    amp = Amplifier(debug=True)
+    r = amp.initialize_configuration(Tici().get_device_type())
+    assert r
+    assert self._check_for_i2c_errors(False)
+
+  def test_shutdown(self):
+    amp = Amplifier(debug=True)
+    for _ in range(10):
+      r = amp.set_global_shutdown(True)
+      r = amp.set_global_shutdown(False)
+      # amp config should be successful, with no i2c errors
+      assert r
+      assert self._check_for_i2c_errors(False)
+
+  def test_init_while_siren_play(self):
+    for _ in range(10):
+      self.panda.set_siren(False)
+      time.sleep(0.1)
+
+      self.panda.set_siren(True)
+      time.sleep(random.randint(0, 5))
+
+      amp = Amplifier(debug=True)
+      r = amp.initialize_configuration(Tici().get_device_type())
+      assert r
+
+      if self._check_for_i2c_errors(True):
+        break
+    else:
+      pytest.fail("didn't hit any i2c errors")

system/hardware/tici/tests/test_power_draw.py

@@ -0,0 +1,128 @@
+from collections import defaultdict, deque
+import pytest
+import time
+import numpy as np
+from dataclasses import dataclass
+from tabulate import tabulate
+
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+from opendbc.car.car_helpers import get_demo_car_params
+from openpilot.common.mock import mock_messages
+from openpilot.common.params import Params
+from openpilot.system.hardware.tici.power_monitor import get_power
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.manager.manager import manager_cleanup
+
+SAMPLE_TIME = 8       # seconds to sample power
+MAX_WARMUP_TIME = 30  # seconds to wait for SAMPLE_TIME consecutive valid samples
+
+@dataclass
+class Proc:
+  procs: list[str]
+  power: float
+  msgs: list[str]
+  rtol: float = 0.05
+  atol: float = 0.12
+
+  @property
+  def name(self):
+    return '+'.join(self.procs)
+
+
+PROCS = [
+  Proc(['camerad'], 1.75, msgs=['roadCameraState', 'wideRoadCameraState', 'driverCameraState']),
+  Proc(['modeld'], 1.12, atol=0.2, msgs=['modelV2']),
+  Proc(['dmonitoringmodeld'], 0.6, msgs=['driverStateV2']),
+  Proc(['encoderd'], 0.23, msgs=[]),
+]
+
+
+@pytest.mark.tici
+class TestPowerDraw:
+
+  def setup_method(self):
+    Params().put("CarParams", get_demo_car_params().to_bytes())
+
+    # wait a bit for power save to disable
+    time.sleep(5)
+
+  def teardown_method(self):
+    manager_cleanup()
+
+  def get_expected_messages(self, proc):
+    return int(sum(SAMPLE_TIME * SERVICE_LIST[msg].frequency for msg in proc.msgs))
+
+  def valid_msg_count(self, proc, msg_counts):
+    msgs_received = sum(msg_counts[msg] for msg in proc.msgs)
+    msgs_expected = self.get_expected_messages(proc)
+    return np.core.numeric.isclose(msgs_expected, msgs_received, rtol=.02, atol=2)
+
+  def valid_power_draw(self, proc, used):
+    return np.core.numeric.isclose(used, proc.power, rtol=proc.rtol, atol=proc.atol)
+
+  def tabulate_msg_counts(self, msgs_and_power):
+    msg_counts = defaultdict(int)
+    for _, counts in msgs_and_power:
+      for msg, count in counts.items():
+        msg_counts[msg] += count
+    return msg_counts
+
+  def get_power_with_warmup_for_target(self, proc, prev):
+    socks = {msg: messaging.sub_sock(msg) for msg in proc.msgs}
+    for sock in socks.values():
+      messaging.drain_sock_raw(sock)
+
+    msgs_and_power = deque([], maxlen=SAMPLE_TIME)
+
+    start_time = time.monotonic()
+
+    while (time.monotonic() - start_time) < MAX_WARMUP_TIME:
+      power = get_power(1)
+      iteration_msg_counts = {}
+      for msg,sock in socks.items():
+        iteration_msg_counts[msg] = len(messaging.drain_sock_raw(sock))
+      msgs_and_power.append((power, iteration_msg_counts))
+
+      if len(msgs_and_power) < SAMPLE_TIME:
+        continue
+
+      msg_counts = self.tabulate_msg_counts(msgs_and_power)
+      now = np.mean([m[0] for m in msgs_and_power])
+
+      if self.valid_msg_count(proc, msg_counts) and self.valid_power_draw(proc, now - prev):
+        break
+
+    return now, msg_counts, time.monotonic() - start_time - SAMPLE_TIME
+
+  @mock_messages(['livePose'])
+  def test_camera_procs(self, subtests):
+    baseline = get_power()
+
+    prev = baseline
+    used = {}
+    warmup_time = {}
+    msg_counts = {}
+
+    for proc in PROCS:
+      for p in proc.procs:
+        managed_processes[p].start()
+      now, local_msg_counts, warmup_time[proc.name] = self.get_power_with_warmup_for_target(proc, prev)
+      msg_counts.update(local_msg_counts)
+
+      used[proc.name] = now - prev
+      prev = now
+
+    manager_cleanup()
+
+    tab = [['process', 'expected (W)', 'measured (W)', '# msgs expected', '# msgs received', "warmup time (s)"]]
+    for proc in PROCS:
+      cur = used[proc.name]
+      expected = proc.power
+      msgs_received = sum(msg_counts[msg] for msg in proc.msgs)
+      tab.append([proc.name, round(expected, 2), round(cur, 2), self.get_expected_messages(proc), msgs_received, round(warmup_time[proc.name], 2)])
+      with subtests.test(proc=proc.name):
+        assert self.valid_msg_count(proc, msg_counts), f"expected {self.get_expected_messages(proc)} msgs, got {msgs_received} msgs"
+        assert self.valid_power_draw(proc, cur), f"expected {expected:.2f}W, got {cur:.2f}W"
+    print(tabulate(tab))
+    print(f"Baseline {baseline:.2f}W\n")

system/loggerd/config.py

@@ -0,0 +1,29 @@
+import os
+from openpilot.system.hardware.hw import Paths
+
+
+CAMERA_FPS = 20
+SEGMENT_LENGTH = 60
+
+STATS_DIR_FILE_LIMIT = 10000
+STATS_SOCKET = "ipc:///tmp/stats"
+STATS_FLUSH_TIME_S = 60
+
+def get_available_percent(default: float) -> float:
+  try:
+    statvfs = os.statvfs(Paths.log_root())
+    available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
+  except OSError:
+    available_percent = default
+
+  return available_percent
+
+
+def get_available_bytes(default: int) -> int:
+  try:
+    statvfs = os.statvfs(Paths.log_root())
+    available_bytes = statvfs.f_bavail * statvfs.f_frsize
+  except OSError:
+    available_bytes = default
+
+  return available_bytes

system/loggerd/deleter.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import threading
+from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
+from openpilot.system.loggerd.uploader import listdir_by_creation
+from openpilot.system.loggerd.xattr_cache import getxattr
+
+MIN_BYTES = 5 * 1024 * 1024 * 1024
+MIN_PERCENT = 30
+
+DELETE_LAST = ['boot', 'crash']
+
+PRESERVE_ATTR_NAME = 'user.preserve'
+PRESERVE_ATTR_VALUE = b'1'
+PRESERVE_COUNT = 5
+
+
+def has_preserve_xattr(d: str) -> bool:
+  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+
+def get_preserved_segments(dirs_by_creation: list[str]) -> set[str]:
+  # skip deleting most recent N preserved segments (and their prior segment)
+  preserved = set()
+  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
+    if n == PRESERVE_COUNT:
+      break
+    date_str, _, seg_str = d.rpartition("--")
+
+    # ignore non-segment directories
+    if not date_str:
+      continue
+    try:
+      seg_num = int(seg_str)
+    except ValueError:
+      continue
+
+    # preserve segment and two prior
+    for _seg_num in range(max(0, seg_num - 2), seg_num + 1):
+      preserved.add(f"{date_str}--{_seg_num}")
+
+  return preserved
+
+
+def deleter_thread(exit_event: threading.Event):
+  while not exit_event.is_set():
+    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+
+    if out_of_percent or out_of_bytes:
+      dirs = listdir_by_creation(Paths.log_root())
+      preserved_dirs = get_preserved_segments(dirs)
+
+      # remove the earliest directory we can
+      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
+        delete_path = os.path.join(Paths.log_root(), delete_dir)
+
+        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+          continue
+
+        try:
+          cloudlog.info(f"deleting {delete_path}")
+          shutil.rmtree(delete_path)
+          break
+        except OSError:
+          cloudlog.exception(f"issue deleting {delete_path}")
+      exit_event.wait(.1)
+    else:
+      exit_event.wait(30)
+
+
+def main():
+  deleter_thread(threading.Event())
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/encoder/encoder.h

@@ -0,0 +1,42 @@
+#pragma once
+
+// has to be in this order
+#ifdef __linux__
+#include "third_party/linux/include/v4l2-controls.h"
+#include <linux/videodev2.h>
+#else
+#define V4L2_BUF_FLAG_KEYFRAME 8
+#endif
+
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionipc.h"
+#include "common/queue.h"
+#include "system/loggerd/loggerd.h"
+
+class VideoEncoder {
+public:
+  VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  virtual ~VideoEncoder() {}
+  virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
+  virtual void encoder_open() = 0;
+  virtual void encoder_close() = 0;
+
+  void publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+
+protected:
+  int in_width, in_height;
+  int out_width, out_height;
+  const EncoderInfo encoder_info;
+
+private:
+  // total frames encoded
+  int cnt = 0;
+  std::unique_ptr<PubMaster> pm;
+  std::vector<capnp::byte> msg_cache;
+};

system/loggerd/encoder/ffmpeg_encoder.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/loggerd.h"
+
+class FfmpegEncoder : public VideoEncoder {
+public:
+  FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  ~FfmpegEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+
+private:
+  int segment_num = -1;
+  int counter = 0;
+  bool is_open = false;
+
+  AVCodecContext *codec_ctx;
+  AVFrame *frame = NULL;
+  std::vector<uint8_t> convert_buf;
+  std::vector<uint8_t> downscale_buf;
+};

system/loggerd/encoder/jpeg_encoder.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstddef>
+#include <cstdint>
+#include <jpeglib.h>
+#include <vector>
+#include <memory>
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionbuf.h"
+
+class JpegEncoder {
+public:
+  JpegEncoder(const std::string &pusblish_name, int width, int height);
+  ~JpegEncoder();
+  void pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra);
+
+private:
+  void generateThumbnail(const uint8_t *y, const uint8_t *uv, int width, int height, int stride);
+  void compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane);
+
+  int thumbnail_width;
+  int thumbnail_height;
+  std::string publish_name;
+  std::vector<uint8_t> yuv_buffer;
+  std::unique_ptr<PubMaster> pm;
+
+  // JPEG output buffer
+  unsigned char* out_buffer = nullptr;
+  unsigned long out_size = 0;
+};

system/loggerd/encoder/v4l_encoder.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "common/queue.h"
+#include "system/loggerd/encoder/encoder.h"
+
+#define BUF_IN_COUNT 7
+#define BUF_OUT_COUNT 6
+
+class V4LEncoder : public VideoEncoder {
+public:
+  V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  ~V4LEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+
+private:
+  int fd;
+
+  bool is_open = false;
+  int segment_num = -1;
+  int counter = 0;
+
+  SafeQueue<VisionIpcBufExtra> extras;
+
+  static void dequeue_handler(V4LEncoder *e);
+  std::thread dequeue_handler_thread;
+
+  VisionBuf buf_out[BUF_OUT_COUNT];
+  SafeQueue<unsigned int> free_buf_in;
+};

system/loggerd/logger.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cassert>
+#include <memory>
+#include <string>
+
+#include "cereal/messaging/messaging.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+#include "system/loggerd/zstd_writer.h"
+
+constexpr int LOG_COMPRESSION_LEVEL = 10;
+
+typedef cereal::Sentinel::SentinelType SentinelType;
+
+class LoggerState {
+public:
+  LoggerState(const std::string& log_root = Path::log_root());
+  ~LoggerState();
+  bool next();
+  void write(uint8_t* data, size_t size, bool in_qlog);
+  inline int segment() const { return part; }
+  inline const std::string& segmentPath() const { return segment_path; }
+  inline const std::string& routeName() const { return route_name; }
+  inline void write(kj::ArrayPtr<kj::byte> bytes, bool in_qlog) { write(bytes.begin(), bytes.size(), in_qlog); }
+  inline void setExitSignal(int signal) { exit_signal = signal; }
+
+protected:
+  int part = -1, exit_signal = 0;
+  std::string route_path, route_name, segment_path, lock_file;
+  kj::Array<capnp::word> init_data;
+  std::unique_ptr<ZstdFileWriter> rlog, qlog;
+};
+
+kj::Array<capnp::word> logger_build_init_data();
+std::string logger_get_identifier(std::string key);
+std::string zstd_decompress(const std::string &in);

system/loggerd/loggerd.h

@@ -0,0 +1,153 @@
+#pragma once
+
+#include <vector>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/services.h"
+#include "msgq/visionipc/visionipc_client.h"
+#include "system/hardware/hw.h"
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+
+#include "system/loggerd/logger.h"
+
+constexpr int MAIN_FPS = 20;
+const int MAIN_BITRATE = 1e7;
+const int LIVESTREAM_BITRATE = 1e6;
+const int QCAM_BITRATE = 256000;
+
+#define NO_CAMERA_PATIENCE 500  // fall back to time-based rotation if all cameras are dead
+
+#define INIT_ENCODE_FUNCTIONS(encode_type)                                \
+  .get_encode_data_func = &cereal::Event::Reader::get##encode_type##Data, \
+  .set_encode_idx_func = &cereal::Event::Builder::set##encode_type##Idx,  \
+  .init_encode_data_func = &cereal::Event::Builder::init##encode_type##Data
+
+const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
+const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
+
+constexpr char PRESERVE_ATTR_NAME[] = "user.preserve";
+constexpr char PRESERVE_ATTR_VALUE = '1';
+class EncoderInfo {
+public:
+  const char *publish_name;
+  const char *thumbnail_name = NULL;
+  const char *filename = NULL;
+  bool record = true;
+  bool include_audio = false;
+  int frame_width = -1;
+  int frame_height = -1;
+  int fps = MAIN_FPS;
+  int bitrate = MAIN_BITRATE;
+  cereal::EncodeIndex::Type encode_type = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS
+                                                         : cereal::EncodeIndex::Type::FULL_H_E_V_C;
+  ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
+  void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
+  cereal::EncodeData::Builder (cereal::Event::Builder::*init_encode_data_func)();
+};
+
+class LogCameraInfo {
+public:
+  const char *thread_name;
+  int fps = MAIN_FPS;
+  VisionStreamType stream_type;
+  std::vector<EncoderInfo> encoder_infos;
+};
+
+const EncoderInfo main_road_encoder_info = {
+  .publish_name = "roadEncodeData",
+  .thumbnail_name = "thumbnail",
+  .filename = "fcamera.hevc",
+  .record = Params().getInt("RecordRoadCam") > 0,
+  INIT_ENCODE_FUNCTIONS(RoadEncode),
+};
+
+const EncoderInfo main_wide_road_encoder_info = {
+  .publish_name = "wideRoadEncodeData",
+  .filename = "ecamera.hevc",
+  .record = Params().getInt("RecordRoadCam") > 1,
+  INIT_ENCODE_FUNCTIONS(WideRoadEncode),
+};
+
+const EncoderInfo main_driver_encoder_info = {
+  .publish_name = "driverEncodeData",
+  .filename = "dcamera.hevc",
+  .record = Params().getBool("RecordFront"),
+  INIT_ENCODE_FUNCTIONS(DriverEncode),
+};
+
+const EncoderInfo stream_road_encoder_info = {
+  .publish_name = "livestreamRoadEncodeData",
+  //.thumbnail_name = "thumbnail",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamRoadEncode),
+};
+
+const EncoderInfo stream_wide_road_encoder_info = {
+  .publish_name = "livestreamWideRoadEncodeData",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamWideRoadEncode),
+};
+
+const EncoderInfo stream_driver_encoder_info = {
+  .publish_name = "livestreamDriverEncodeData",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamDriverEncode),
+};
+
+const EncoderInfo qcam_encoder_info = {
+  .publish_name = "qRoadEncodeData",
+  .filename = "qcamera.ts",
+  .bitrate = QCAM_BITRATE,
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .frame_width = 526,
+  .frame_height = 330,
+  .include_audio = Params().getBool("RecordAudio"),
+  INIT_ENCODE_FUNCTIONS(QRoadEncode),
+};
+
+const LogCameraInfo road_camera_info{
+  .thread_name = "road_cam_encoder",
+  .stream_type = VISION_STREAM_ROAD,
+  .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+};
+
+const LogCameraInfo wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {main_wide_road_encoder_info}
+};
+
+const LogCameraInfo driver_camera_info{
+  .thread_name = "driver_cam_encoder",
+  .stream_type = VISION_STREAM_DRIVER,
+  .encoder_infos = {main_driver_encoder_info}
+};
+
+const LogCameraInfo stream_road_camera_info{
+  .thread_name = "road_cam_encoder",
+  .stream_type = VISION_STREAM_ROAD,
+  .encoder_infos = {stream_road_encoder_info}
+};
+
+const LogCameraInfo stream_wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {stream_wide_road_encoder_info}
+};
+
+const LogCameraInfo stream_driver_camera_info{
+  .thread_name = "driver_cam_encoder",
+  .stream_type = VISION_STREAM_DRIVER,
+  .encoder_infos = {stream_driver_encoder_info}
+};
+
+const LogCameraInfo cameras_logged[] = {road_camera_info, wide_road_camera_info};//, driver_camera_info};
+const LogCameraInfo stream_cameras_logged[] = {stream_road_camera_info, stream_wide_road_camera_info};//, stream_driver_camera_info};

system/loggerd/tests/loggerd_tests_common.py

@@ -0,0 +1,90 @@
+import os
+import random
+from pathlib import Path
+
+
+import openpilot.system.loggerd.deleter as deleter
+import openpilot.system.loggerd.uploader as uploader
+from openpilot.common.params import Params
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import setxattr
+
+
+def create_random_file(file_path: Path, size_mb: float, lock: bool = False, upload_xattr: bytes = None) -> None:
+  file_path.parent.mkdir(parents=True, exist_ok=True)
+
+  if lock:
+    lock_path = str(file_path) + ".lock"
+    os.close(os.open(lock_path, os.O_CREAT | os.O_EXCL))
+
+  chunks = 128
+  chunk_bytes = int(size_mb * 1024 * 1024 / chunks)
+  data = os.urandom(chunk_bytes)
+
+  with open(file_path, "wb") as f:
+    for _ in range(chunks):
+      f.write(data)
+
+  if upload_xattr is not None:
+    setxattr(str(file_path), uploader.UPLOAD_ATTR_NAME, upload_xattr)
+
+class MockResponse:
+  def __init__(self, text, status_code):
+    self.text = text
+    self.status_code = status_code
+
+class MockApi:
+  def __init__(self, dongle_id):
+    pass
+
+  def get(self, *args, **kwargs):
+    return MockResponse('{"url": "http://localhost/does/not/exist", "headers": {}}', 200)
+
+  def get_token(self):
+    return "fake-token"
+
+class MockApiIgnore:
+  def __init__(self, dongle_id):
+    pass
+
+  def get(self, *args, **kwargs):
+    return MockResponse('', 412)
+
+  def get_token(self):
+    return "fake-token"
+
+class UploaderTestCase:
+  f_type = "UNKNOWN"
+
+  root: Path
+  seg_num: int
+  seg_format: str
+  seg_format2: str
+  seg_dir: str
+
+  def set_ignore(self):
+    uploader.Api = MockApiIgnore
+
+  def setup_method(self):
+    uploader.Api = MockApi
+    uploader.fake_upload = True
+    uploader.force_wifi = True
+    uploader.allow_sleep = False
+    self.seg_num = random.randint(1, 300)
+    self.seg_format = "00000004--0ac3964c96--{}"
+    self.seg_format2 = "00000005--4c4e99b08b--{}"
+    self.seg_dir = self.seg_format.format(self.seg_num)
+
+    self.params = Params()
+    self.params.put("IsOffroad", "1")
+    self.params.put("DongleId", "0000000000000000")
+
+  def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False,
+                          upload_xattr: bytes = None, preserve_xattr: bytes = None) -> Path:
+    file_path = Path(Paths.log_root()) / f_dir / fn
+    create_random_file(file_path, size_mb, lock, upload_xattr)
+
+    if preserve_xattr is not None:
+      setxattr(str(file_path.parent), deleter.PRESERVE_ATTR_NAME, preserve_xattr)
+
+    return file_path

system/loggerd/tests/test_deleter.py

@@ -0,0 +1,117 @@
+import time
+import threading
+from collections import namedtuple
+from pathlib import Path
+from collections.abc import Sequence
+
+import openpilot.system.loggerd.deleter as deleter
+from openpilot.common.timeout import Timeout, TimeoutException
+from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase
+
+Stats = namedtuple("Stats", ['f_bavail', 'f_blocks', 'f_frsize'])
+
+
+class TestDeleter(UploaderTestCase):
+  def fake_statvfs(self, d):
+    return self.fake_stats
+
+  def setup_method(self):
+    self.f_type = "fcamera.hevc"
+    super().setup_method()
+    self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096)
+    deleter.os.statvfs = self.fake_statvfs
+
+  def start_thread(self):
+    self.end_event = threading.Event()
+    self.del_thread = threading.Thread(target=deleter.deleter_thread, args=[self.end_event])
+    self.del_thread.daemon = True
+    self.del_thread.start()
+
+  def join_thread(self):
+    self.end_event.set()
+    self.del_thread.join()
+
+  def test_delete(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type, 1)
+
+    self.start_thread()
+
+    try:
+      with Timeout(2, "Timeout waiting for file to be deleted"):
+        while f_path.exists():
+          time.sleep(0.01)
+    finally:
+      self.join_thread()
+
+  def assertDeleteOrder(self, f_paths: Sequence[Path], timeout: int = 5) -> None:
+    deleted_order = []
+
+    self.start_thread()
+    try:
+      with Timeout(timeout, "Timeout waiting for files to be deleted"):
+        while True:
+          for f in f_paths:
+            if not f.exists() and f not in deleted_order:
+              deleted_order.append(f)
+          if len(deleted_order) == len(f_paths):
+            break
+          time.sleep(0.01)
+    except TimeoutException:
+      print("Not deleted:", [f for f in f_paths if f not in deleted_order])
+      raise
+    finally:
+      self.join_thread()
+
+    assert deleted_order == f_paths, "Files not deleted in expected order"
+
+  def test_delete_order(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+    ])
+
+  def test_delete_many_preserved(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data(self.seg_format.format(2), self.f_type),
+    ] + [
+      self.make_file_with_data(self.seg_format2.format(i), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE)
+      for i in range(5)
+    ])
+
+  def test_delete_last(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(0), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data("boot", self.seg_format[:-4]),
+      self.make_file_with_data("crash", self.seg_format2[:-4]),
+    ])
+
+  def test_no_delete_when_available_space(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type)
+
+    block_size = 4096
+    available = (10 * 1024 * 1024 * 1024) / block_size  # 10GB free
+    self.fake_stats = Stats(f_bavail=available, f_blocks=10, f_frsize=block_size)
+
+    self.start_thread()
+    start_time = time.monotonic()
+    while f_path.exists() and time.monotonic() - start_time < 2:
+      time.sleep(0.01)
+    self.join_thread()
+
+    assert f_path.exists(), "File deleted with available space"
+
+  def test_no_delete_with_lock_file(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
+
+    self.start_thread()
+    start_time = time.monotonic()
+    while f_path.exists() and time.monotonic() - start_time < 2:
+      time.sleep(0.01)
+    self.join_thread()
+
+    assert f_path.exists(), "File deleted when locked"

system/loggerd/tests/test_encoder.py

@@ -0,0 +1,150 @@
+import math
+import os
+import pytest
+import random
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from parameterized import parameterized
+from tqdm import trange
+
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.hardware import TICI
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.system.hardware.hw import Paths
+
+SEGMENT_LENGTH = 2
+FULL_SIZE = 2507572
+CAMERAS = [
+  ("fcamera.hevc", 20, FULL_SIZE, "roadEncodeIdx"),
+  ("dcamera.hevc", 20, FULL_SIZE, "driverEncodeIdx"),
+  ("ecamera.hevc", 20, FULL_SIZE, "wideRoadEncodeIdx"),
+  ("qcamera.ts", 20, 130000, None),
+]
+
+# we check frame count, so we don't have to be too strict on size
+FILE_SIZE_TOLERANCE = 0.7
+
+
+@pytest.mark.tici # TODO: all of loggerd should work on PC
+class TestEncoder:
+
+  def setup_method(self):
+    self._clear_logs()
+    os.environ["LOGGERD_TEST"] = "1"
+    os.environ["LOGGERD_SEGMENT_LENGTH"] = str(SEGMENT_LENGTH)
+
+  def teardown_method(self):
+    self._clear_logs()
+
+  def _clear_logs(self):
+    if os.path.exists(Paths.log_root()):
+      shutil.rmtree(Paths.log_root())
+
+  def _get_latest_segment_path(self):
+    last_route = sorted(Path(Paths.log_root()).iterdir())[-1]
+    return os.path.join(Paths.log_root(), last_route)
+
+  # TODO: this should run faster than real time
+  @parameterized.expand([(True, ), (False, )])
+  def test_log_rotation(self, record_front):
+    Params().put_bool("RecordFront", record_front)
+
+    managed_processes['sensord'].start()
+    managed_processes['loggerd'].start()
+    managed_processes['encoderd'].start()
+
+    time.sleep(1.0)
+    managed_processes['camerad'].start()
+
+    num_segments = int(os.getenv("SEGMENTS", random.randint(2, 8)))
+
+    # wait for loggerd to make the dir for first segment
+    route_prefix_path = None
+    with Timeout(int(SEGMENT_LENGTH*3)):
+      while route_prefix_path is None:
+        try:
+          route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0]
+        except Exception:
+          time.sleep(0.1)
+
+    def check_seg(i):
+      # check each camera file size
+      counts = []
+      first_frames = []
+      for camera, fps, size, encode_idx_name in CAMERAS:
+        if not record_front and "dcamera" in camera:
+          continue
+
+        file_path = f"{route_prefix_path}--{i}/{camera}"
+
+        # check file exists
+        assert os.path.exists(file_path), f"segment #{i}: '{file_path}' missing"
+
+        # TODO: this ffprobe call is really slow
+        # check frame count
+        cmd = f"ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 {file_path}"
+        if TICI:
+          cmd = "LD_LIBRARY_PATH=/usr/local/lib " + cmd
+
+        expected_frames = fps * SEGMENT_LENGTH
+        probe = subprocess.check_output(cmd, shell=True, encoding='utf8')
+        frame_count = int(probe.split('\n')[0].strip())
+        counts.append(frame_count)
+
+        assert frame_count == expected_frames, \
+                         f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}"
+
+        # sanity check file size
+        file_size = os.path.getsize(file_path)
+        assert math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE), \
+                        f"{file_path} size {file_size} isn't close to target size {size}"
+
+        # Check encodeIdx
+        if encode_idx_name is not None:
+          rlog_path = f"{route_prefix_path}--{i}/rlog.zst"
+          msgs = [m for m in LogReader(rlog_path) if m.which() == encode_idx_name]
+          encode_msgs = [getattr(m, encode_idx_name) for m in msgs]
+
+          valid = [m.valid for m in msgs]
+          segment_idxs = [m.segmentId for m in encode_msgs]
+          encode_idxs = [m.encodeId for m in encode_msgs]
+          frame_idxs = [m.frameId for m in encode_msgs]
+
+          # Check frame count
+          assert frame_count == len(segment_idxs)
+          assert frame_count == len(encode_idxs)
+
+          # Check for duplicates or skips
+          assert 0 == segment_idxs[0]
+          assert len(set(segment_idxs)) == len(segment_idxs)
+
+          assert all(valid)
+
+          assert expected_frames * i == encode_idxs[0]
+          first_frames.append(frame_idxs[0])
+          assert len(set(encode_idxs)) == len(encode_idxs)
+
+      assert 1 == len(set(first_frames))
+
+      if TICI:
+        expected_frames = fps * SEGMENT_LENGTH
+        assert min(counts) == expected_frames
+      shutil.rmtree(f"{route_prefix_path}--{i}")
+
+    try:
+      for i in trange(num_segments):
+        # poll for next segment
+        with Timeout(int(SEGMENT_LENGTH*10), error_msg=f"timed out waiting for segment {i}"):
+          while Path(f"{route_prefix_path}--{i+1}") not in Path(Paths.log_root()).iterdir():
+            time.sleep(0.1)
+        check_seg(i)
+    finally:
+      managed_processes['loggerd'].stop()
+      managed_processes['encoderd'].stop()
+      managed_processes['camerad'].stop()
+      managed_processes['sensord'].stop()

system/loggerd/tests/test_loggerd.py

@@ -0,0 +1,283 @@
+import numpy as np
+import os
+import re
+import random
+import string
+import subprocess
+import time
+from collections import defaultdict
+from pathlib import Path
+import pytest
+
+import cereal.messaging as messaging
+from cereal import log
+from cereal.services import SERVICE_LIST
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import getxattr
+from openpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.version import get_version
+from openpilot.tools.lib.helpers import RE
+from openpilot.tools.lib.logreader import LogReader
+from msgq.visionipc import VisionIpcServer, VisionStreamType
+from openpilot.common.transformations.camera import DEVICE_CAMERAS
+
+SentinelType = log.Sentinel.SentinelType
+
+CEREAL_SERVICES = [f for f in log.Event.schema.union_fields if f in SERVICE_LIST
+                   and SERVICE_LIST[f].should_log and "encode" not in f.lower()]
+
+
+class TestLoggerd:
+  def _get_latest_log_dir(self):
+    log_dirs = sorted(Path(Paths.log_root()).iterdir(), key=lambda f: f.stat().st_mtime)
+    return log_dirs[-1]
+
+  def _get_log_dir(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_dir():
+          return path
+    return None
+
+  def _get_log_fn(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_file():
+          return path
+    return None
+
+  def _gen_bootlog(self):
+    with Timeout(5):
+      out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), encoding='utf-8')
+
+    log_fn = self._get_log_fn(out)
+
+    # check existence
+    assert log_fn is not None
+
+    return log_fn
+
+  def _check_init_data(self, msgs):
+    msg = msgs[0]
+    assert msg.which() == 'initData'
+
+  def _check_sentinel(self, msgs, route):
+    start_type = SentinelType.startOfRoute if route else SentinelType.startOfSegment
+    assert msgs[1].sentinel.type == start_type
+
+    end_type = SentinelType.endOfRoute if route else SentinelType.endOfSegment
+    assert msgs[-1].sentinel.type == end_type
+
+  def _publish_random_messages(self, services: list[str]) -> dict[str, list]:
+    pm = messaging.PubMaster(services)
+
+    managed_processes["loggerd"].start()
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+
+    sent_msgs = defaultdict(list)
+    for _ in range(random.randint(2, 10) * 100):
+      for s in services:
+        try:
+          m = messaging.new_message(s)
+        except Exception:
+          m = messaging.new_message(s, random.randint(2, 10))
+        pm.send(s, m)
+        sent_msgs[s].append(m)
+
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+    managed_processes["loggerd"].stop()
+
+    return sent_msgs
+
+  def test_init_data_values(self):
+    os.environ["CLEAN"] = random.choice(["0", "1"])
+
+    dongle  = ''.join(random.choice(string.printable) for n in range(random.randint(1, 100)))
+    fake_params = [
+      # param, initData field, value
+      ("DongleId", "dongleId", dongle),
+      ("GitCommit", "gitCommit", "commit"),
+      ("GitCommitDate", "gitCommitDate", "date"),
+      ("GitBranch", "gitBranch", "branch"),
+      ("GitRemote", "gitRemote", "remote"),
+    ]
+    params = Params()
+    for k, _, v in fake_params:
+      params.put(k, v)
+    params.put("AccessToken", "abc")
+
+    lr = list(LogReader(str(self._gen_bootlog())))
+    initData = lr[0].initData
+
+    assert initData.dirty != bool(os.environ["CLEAN"])
+    assert initData.version == get_version()
+
+    if os.path.isfile("/proc/cmdline"):
+      with open("/proc/cmdline") as f:
+        assert list(initData.kernelArgs) == f.read().strip().split(" ")
+
+      with open("/proc/version") as f:
+        assert initData.kernelVersion == f.read()
+
+    # check params
+    logged_params = {entry.key: entry.value for entry in initData.params.entries}
+    expected_params = {k for k, _, __ in fake_params} | {'AccessToken', 'BootCount'}
+    assert set(logged_params.keys()) == expected_params, set(logged_params.keys()) ^ expected_params
+    assert logged_params['AccessToken'] == b'', f"DONT_LOG param value was logged: {repr(logged_params['AccessToken'])}"
+    for param_key, initData_key, v in fake_params:
+      assert getattr(initData, initData_key) == v
+      assert logged_params[param_key].decode() == v
+
+  @pytest.mark.skip("FIXME: encoderd sometimes crashes in CI when running with pytest-xdist")
+  def test_rotation(self):
+    os.environ["LOGGERD_TEST"] = "1"
+    Params().put("RecordFront", "1")
+
+    d = DEVICE_CAMERAS[("tici", "ar0231")]
+    expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
+    streams = [(VisionStreamType.VISION_STREAM_ROAD, (d.fcam.width, d.fcam.height, 2048*2346, 2048, 2048*1216), "roadCameraState"),
+               (VisionStreamType.VISION_STREAM_DRIVER, (d.dcam.width, d.dcam.height, 2048*2346, 2048, 2048*1216), "driverCameraState"),
+               (VisionStreamType.VISION_STREAM_WIDE_ROAD, (d.ecam.width, d.ecam.height, 2048*2346, 2048, 2048*1216), "wideRoadCameraState")]
+
+    pm = messaging.PubMaster(["roadCameraState", "driverCameraState", "wideRoadCameraState"])
+    vipc_server = VisionIpcServer("camerad")
+    for stream_type, frame_spec, _ in streams:
+      vipc_server.create_buffers_with_sizes(stream_type, 40, *(frame_spec))
+    vipc_server.start_listener()
+
+    num_segs = random.randint(2, 5)
+    length = random.randint(1, 3)
+    os.environ["LOGGERD_SEGMENT_LENGTH"] = str(length)
+    managed_processes["loggerd"].start()
+    managed_processes["encoderd"].start()
+    assert pm.wait_for_readers_to_update("roadCameraState", timeout=5)
+
+    fps = 20.0
+    for n in range(1, int(num_segs*length*fps)+1):
+      for stream_type, frame_spec, state in streams:
+        dat = np.empty(frame_spec[2], dtype=np.uint8)
+        vipc_server.send(stream_type, dat[:].flatten().tobytes(), n, n/fps, n/fps)
+
+        camera_state = messaging.new_message(state)
+        frame = getattr(camera_state, state)
+        frame.frameId = n
+        pm.send(state, camera_state)
+
+      for _, _, state in streams:
+        assert pm.wait_for_readers_to_update(state, timeout=5, dt=0.001)
+
+    managed_processes["loggerd"].stop()
+    managed_processes["encoderd"].stop()
+
+    route_path = str(self._get_latest_log_dir()).rsplit("--", 1)[0]
+    for n in range(num_segs):
+      p = Path(f"{route_path}--{n}")
+      logged = {f.name for f in p.iterdir() if f.is_file()}
+      diff = logged ^ expected_files
+      assert len(diff) == 0, f"didn't get all expected files. run={_} seg={n} {route_path=}, {diff=}\n{logged=} {expected_files=}"
+
+  def test_bootlog(self):
+    # generate bootlog with fake launch log
+    launch_log = ''.join(str(random.choice(string.printable)) for _ in range(100))
+    with open("/tmp/launch_log", "w") as f:
+      f.write(launch_log)
+
+    bootlog_path = self._gen_bootlog()
+    lr = list(LogReader(str(bootlog_path)))
+
+    # check length
+    assert len(lr) == 2  # boot + initData
+
+    self._check_init_data(lr)
+
+    # check msgs
+    bootlog_msgs = [m for m in lr if m.which() == 'boot']
+    assert len(bootlog_msgs) == 1
+
+    # sanity check values
+    boot = bootlog_msgs.pop().boot
+    assert abs(boot.wallTimeNanos - time.time_ns()) < 5*1e9 # within 5s
+    assert boot.launchLog == launch_log
+
+    for fn in ["console-ramoops", "pmsg-ramoops-0"]:
+      path = Path(os.path.join("/sys/fs/pstore/", fn))
+      if path.is_file():
+        with open(path, "rb") as f:
+          expected_val = f.read()
+        bootlog_val = [e.value for e in boot.pstore.entries if e.key == fn][0]
+        assert expected_val == bootlog_val
+
+    # next one should increment by one
+    bl1 = re.match(RE.LOG_ID_V2, bootlog_path.name)
+    bl2 = re.match(RE.LOG_ID_V2, self._gen_bootlog().name)
+    assert bl1.group('uid') != bl2.group('uid')
+    assert int(bl1.group('count')) == 0 and int(bl2.group('count')) == 1
+
+  def test_qlog(self):
+    qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is not None]
+    no_qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is None]
+
+    services = random.sample(qlog_services, random.randint(2, min(10, len(qlog_services)))) + \
+               random.sample(no_qlog_services, random.randint(2, min(10, len(no_qlog_services))))
+    sent_msgs = self._publish_random_messages(services)
+
+    qlog_path = os.path.join(self._get_latest_log_dir(), "qlog.zst")
+    lr = list(LogReader(qlog_path))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    recv_msgs = defaultdict(list)
+    for m in lr:
+      recv_msgs[m.which()].append(m)
+
+    for s, msgs in sent_msgs.items():
+      recv_cnt = len(recv_msgs[s])
+
+      if s in no_qlog_services:
+        # check services with no specific decimation aren't in qlog
+        assert recv_cnt == 0, f"got {recv_cnt} {s} msgs in qlog"
+      else:
+        # check logged message count matches decimation
+        expected_cnt = (len(msgs) - 1) // SERVICE_LIST[s].decimation + 1
+        assert recv_cnt == expected_cnt, f"expected {expected_cnt} msgs for {s}, got {recv_cnt}"
+
+  def test_rlog(self):
+    services = random.sample(CEREAL_SERVICES, random.randint(5, 10))
+    sent_msgs = self._publish_random_messages(services)
+
+    lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog.zst")))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    # check all messages were logged and in order
+    lr = lr[2:-1] # slice off initData and both sentinels
+    for m in lr:
+      sent = sent_msgs[m.which()].pop(0)
+      sent.clear_write_flag()
+      assert sent.to_bytes() == m.as_builder().to_bytes()
+
+  def test_preserving_flagged_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) | {"userFlag"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+  def test_not_preserving_unflagged_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) - {"userFlag"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) is None

system/loggerd/tests/test_uploader.py

@@ -0,0 +1,184 @@
+import os
+import time
+import threading
+import logging
+import json
+from pathlib import Path
+from openpilot.system.hardware.hw import Paths
+
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.loggerd.uploader import main, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
+
+from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase
+
+
+class FakeLogHandler(logging.Handler):
+  def __init__(self):
+    logging.Handler.__init__(self)
+    self.reset()
+
+  def reset(self):
+    self.upload_order = list()
+    self.upload_ignored = list()
+
+  def emit(self, record):
+    try:
+      j = json.loads(record.getMessage())
+      if j["event"] == "upload_success":
+        self.upload_order.append(j["key"])
+      if j["event"] == "upload_ignored":
+        self.upload_ignored.append(j["key"])
+    except Exception:
+      pass
+
+log_handler = FakeLogHandler()
+cloudlog.addHandler(log_handler)
+
+
+class TestUploader(UploaderTestCase):
+  def setup_method(self):
+    super().setup_method()
+    log_handler.reset()
+
+  def start_thread(self):
+    self.end_event = threading.Event()
+    self.up_thread = threading.Thread(target=main, args=[self.end_event])
+    self.up_thread.daemon = True
+    self.up_thread.start()
+
+  def join_thread(self):
+    self.end_event.set()
+    self.up_thread.join()
+
+  def gen_files(self, lock=False, xattr: bytes = None, boot=True) -> list[Path]:
+    f_paths = []
+    for t in ["qlog", "rlog", "dcamera.hevc", "fcamera.hevc"]:
+      f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock, upload_xattr=xattr))
+
+    if boot:
+      f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}", 1, lock=lock, upload_xattr=xattr))
+    return f_paths
+
+  def gen_order(self, seg1: list[int], seg2: list[int], boot=True) -> list[str]:
+    keys = []
+    if boot:
+      keys += [f"boot/{self.seg_format.format(i)}.zst" for i in seg1]
+      keys += [f"boot/{self.seg_format2.format(i)}.zst" for i in seg2]
+    keys += [f"{self.seg_format.format(i)}/qlog.zst" for i in seg1]
+    keys += [f"{self.seg_format2.format(i)}/qlog.zst" for i in seg2]
+    return keys
+
+  def test_upload(self):
+    self.gen_files(lock=False)
+
+    self.start_thread()
+    # allow enough time that files could upload twice if there is a bug in the logic
+    time.sleep(1)
+    self.join_thread()
+
+    exp_order = self.gen_order([self.seg_num], [])
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_upload_with_wrong_xattr(self):
+    self.gen_files(lock=False, xattr=b'0')
+
+    self.start_thread()
+    # allow enough time that files could upload twice if there is a bug in the logic
+    time.sleep(1)
+    self.join_thread()
+
+    exp_order = self.gen_order([self.seg_num], [])
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_upload_ignored(self):
+    self.set_ignore()
+    self.gen_files(lock=False)
+
+    self.start_thread()
+    # allow enough time that files could upload twice if there is a bug in the logic
+    time.sleep(1)
+    self.join_thread()
+
+    exp_order = self.gen_order([self.seg_num], [])
+
+    assert len(log_handler.upload_order) == 0, "Some files were not ignored"
+    assert not len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore"
+    assert not len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not ignored"
+
+    assert log_handler.upload_ignored == exp_order, "Files ignored in wrong order"
+
+  def test_upload_files_in_create_order(self):
+    seg1_nums = [0, 1, 2, 10, 20]
+    for i in seg1_nums:
+      self.seg_dir = self.seg_format.format(i)
+      self.gen_files(boot=False)
+    seg2_nums = [5, 50, 51]
+    for i in seg2_nums:
+      self.seg_dir = self.seg_format2.format(i)
+      self.gen_files(boot=False)
+
+    exp_order = self.gen_order(seg1_nums, seg2_nums, boot=False)
+
+    self.start_thread()
+    # allow enough time that files could upload twice if there is a bug in the logic
+    time.sleep(1)
+    self.join_thread()
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_no_upload_with_lock_file(self):
+    self.start_thread()
+
+    time.sleep(0.25)
+    f_paths = self.gen_files(lock=True, boot=False)
+
+    # allow enough time that files should have been uploaded if they would be uploaded
+    time.sleep(1)
+    self.join_thread()
+
+    for f_path in f_paths:
+      fn = f_path.with_suffix(f_path.suffix.replace(".zst", ""))
+      uploaded = UPLOAD_ATTR_NAME in os.listxattr(fn) and os.getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
+      assert not uploaded, "File upload when locked"
+
+  def test_no_upload_with_xattr(self):
+    self.gen_files(lock=False, xattr=UPLOAD_ATTR_VALUE)
+
+    self.start_thread()
+    # allow enough time that files could upload twice if there is a bug in the logic
+    time.sleep(1)
+    self.join_thread()
+
+    assert len(log_handler.upload_order) == 0, "File uploaded again"
+
+  def test_clear_locks_on_startup(self):
+    f_paths = self.gen_files(lock=True, boot=False)
+    self.start_thread()
+    time.sleep(0.25)
+    self.join_thread()
+
+    for f_path in f_paths:
+      lock_path = f_path.with_suffix(f_path.suffix + ".lock")
+      assert not lock_path.is_file(), "File lock not cleared on startup"

system/loggerd/uploader.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+import json
+import os
+import random
+import requests
+import threading
+import time
+import traceback
+import datetime
+from collections.abc import Iterator
+
+from cereal import log
+import cereal.messaging as messaging
+from openpilot.common.api import Api
+from openpilot.common.file_helpers import get_upload_stream
+from openpilot.common.params import Params
+from openpilot.common.realtime import set_core_affinity
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.common.swaglog import cloudlog
+
+NetworkType = log.DeviceState.NetworkType
+UPLOAD_ATTR_NAME = 'user.upload'
+UPLOAD_ATTR_VALUE = b'1'
+
+MAX_UPLOAD_SIZES = {
+  "qlog": 25*1e6,  # can't be too restrictive here since we use qlogs to find
+                   # bugs, including ones that can cause massive log sizes
+  "qcam": 5*1e6,
+}
+
+allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
+force_wifi = os.getenv("FORCEWIFI") is not None
+fake_upload = os.getenv("FAKEUPLOAD") is not None
+
+
+class FakeRequest:
+  def __init__(self):
+    self.headers = {"Content-Length": "0"}
+
+
+class FakeResponse:
+  def __init__(self):
+    self.status_code = 200
+    self.request = FakeRequest()
+
+
+def get_directory_sort(d: str) -> list[str]:
+  # ensure old format is sorted sooner
+  o = ["0", ] if d.startswith("2024-") else ["1", ]
+  return o + [s.rjust(10, '0') for s in d.rsplit('--', 1)]
+
+def listdir_by_creation(d: str) -> list[str]:
+  if not os.path.isdir(d):
+    return []
+
+  try:
+    paths = [f for f in os.listdir(d) if os.path.isdir(os.path.join(d, f))]
+    paths = sorted(paths, key=get_directory_sort)
+    return paths
+  except OSError:
+    cloudlog.exception("listdir_by_creation failed")
+    return []
+
+def clear_locks(root: str) -> None:
+  for logdir in os.listdir(root):
+    path = os.path.join(root, logdir)
+    try:
+      for fname in os.listdir(path):
+        if fname.endswith(".lock"):
+          os.unlink(os.path.join(path, fname))
+    except OSError:
+      cloudlog.exception("clear_locks failed")
+
+
+class Uploader:
+  def __init__(self, dongle_id: str, root: str):
+    self.dongle_id = dongle_id
+    self.api = Api(dongle_id)
+    self.root = root
+
+    self.params = Params()
+
+    # stats for last successfully uploaded file
+    self.last_filename = ""
+
+    self.immediate_folders = ["crash/", "boot/"]
+    self.immediate_priority = {"qlog": 0, "qlog.zst": 0, "qcamera.ts": 1}
+    #if (self.params.get_int("EnableConnect") == 2):
+    #  self.immediate_priority.update({"rlog": 0, "rlog.zst": 0})
+
+  def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]:
+    r = self.params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
+    requested_routes = [] if r is None else [route for route in r.split(",") if route]
+
+    for logdir in listdir_by_creation(self.root):
+      path = os.path.join(self.root, logdir)
+      try:
+        names = os.listdir(path)
+      except OSError:
+        continue
+
+      if any(name.endswith(".lock") for name in names):
+        continue
+
+      for name in sorted(names, key=lambda n: self.immediate_priority.get(n, 1000)):
+        key = os.path.join(logdir, name)
+        fn = os.path.join(path, name)
+        # skip files already uploaded
+        try:
+          ctime = os.path.getctime(fn)
+          is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
+        except OSError:
+          cloudlog.event("uploader_getxattr_failed", key=key, fn=fn)
+          # deleter could have deleted, so skip
+          continue
+        if is_uploaded:
+          continue
+
+        # limit uploading on metered connections
+        if metered:
+          dt = datetime.timedelta(hours=12)
+          if logdir in self.immediate_folders and (datetime.datetime.now() - datetime.datetime.fromtimestamp(ctime)) < dt:
+            continue
+
+          if name == "qcamera.ts" and not any(logdir.startswith(r.split('|')[-1]) for r in requested_routes):
+            continue
+
+        yield name, key, fn
+
+  def next_file_to_upload(self, metered: bool) -> tuple[str, str, str] | None:
+    upload_files = list(self.list_upload_files(metered))
+
+    for name, key, fn in upload_files:
+      if any(f in fn for f in self.immediate_folders):
+        return name, key, fn
+
+    for name, key, fn in upload_files:
+      if name in self.immediate_priority:
+        return name, key, fn
+
+    return None
+
+  def do_upload(self, key: str, fn: str):
+    url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
+    if url_resp.status_code == 412:
+      return url_resp
+
+    url_resp_json = json.loads(url_resp.text)
+    url = url_resp_json['url']
+    headers = url_resp_json['headers']
+    cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
+
+    if fake_upload:
+      return FakeResponse()
+
+    stream = None
+    try:
+      compress = key.endswith('.zst') and not fn.endswith('.zst')
+      stream, _ = get_upload_stream(fn, compress)
+      response = requests.put(url, data=stream, headers=headers, timeout=10)
+      return response
+    finally:
+      if stream:
+        stream.close()
+
+  def upload(self, name: str, key: str, fn: str, network_type: int, metered: bool) -> bool:
+    try:
+      sz = os.path.getsize(fn)
+    except OSError:
+      cloudlog.exception("upload: getsize failed")
+      return False
+
+    cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if sz == 0:
+      # tag files of 0 size as uploaded
+      success = True
+    elif name in MAX_UPLOAD_SIZES and sz > MAX_UPLOAD_SIZES[name]:
+      cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
+      success = True
+    else:
+      start_time = time.monotonic()
+
+      stat = None
+      last_exc = None
+      try:
+        stat = self.do_upload(key, fn)
+      except Exception as e:
+        last_exc = (e, traceback.format_exc())
+
+      if stat is not None and stat.status_code in (200, 201, 401, 403, 412):
+        self.last_filename = fn
+        dt = time.monotonic() - start_time
+        if stat.status_code == 412:
+          cloudlog.event("upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+        else:
+          content_length = int(stat.request.headers.get("Content-Length", 0))
+          speed = (content_length / 1e6) / dt
+          cloudlog.event("upload_success", key=key, fn=fn, sz=sz, content_length=content_length,
+                         network_type=network_type, metered=metered, speed=speed)
+        success = True
+      else:
+        success = False
+        cloudlog.event("upload_failed", stat=stat, exc=last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if success:
+      # tag file as uploaded
+      try:
+        setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
+      except OSError:
+        cloudlog.event("uploader_setxattr_failed", exc=last_exc, key=key, fn=fn, sz=sz)
+
+    return success
+
+
+  def step(self, network_type: int, metered: bool) -> bool | None:
+    d = self.next_file_to_upload(metered)
+    if d is None:
+      return None
+
+    name, key, fn = d
+
+    # qlogs and bootlogs need to be compressed before uploading
+    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.zst')):
+      key += ".zst"
+
+    return self.upload(name, key, fn, network_type, metered)
+
+
+def main(exit_event: threading.Event = None) -> None:
+  if exit_event is None:
+    exit_event = threading.Event()
+
+  try:
+    set_core_affinity([0, 1, 2, 3])
+  except Exception:
+    cloudlog.exception("failed to set core affinity")
+
+  clear_locks(Paths.log_root())
+
+  params = Params()
+  dongle_id = params.get("DongleId", encoding='utf8')
+
+  if dongle_id is None:
+    cloudlog.info("uploader missing dongle_id")
+    raise Exception("uploader can't start without dongle id")
+
+  sm = messaging.SubMaster(['deviceState'])
+  uploader = Uploader(dongle_id, Paths.log_root())
+
+  backoff = 0.1
+  while not exit_event.is_set():
+    sm.update(0)
+    offroad = params.get_bool("IsOffroad")
+    network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi
+    if network_type == NetworkType.none:
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
+      continue
+
+    success = uploader.step(sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered)
+    if success is None:
+      backoff = 60 if offroad else 5
+    elif success:
+      backoff = 0.1
+    else:
+      cloudlog.info("upload backoff %r", backoff)
+      backoff = min(backoff*2, 120)
+    if allow_sleep:
+      time.sleep(backoff + random.uniform(0, backoff))
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/video_writer.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <string>
+#include <deque>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+#include "cereal/messaging/messaging.h"
+
+class VideoWriter {
+public:
+  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
+  void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
+  void write_audio(uint8_t *data, int len, long long timestamp, int sample_rate);
+
+  ~VideoWriter();
+
+private:
+  void initialize_audio(int sample_rate);
+  void encode_and_write_audio_frame(AVFrame* frame);
+
+  std::string vid_path, lock_path;
+  FILE *of = nullptr;
+
+  AVCodecContext *codec_ctx;
+  AVFormatContext *ofmt_ctx;
+  AVStream *out_stream;
+
+  bool audio_initialized = false;
+  bool header_written = false;
+  AVStream *audio_stream = nullptr;
+  AVCodecContext *audio_codec_ctx = nullptr;
+  AVFrame *audio_frame = nullptr;
+  uint64_t audio_pts = 0;
+  std::deque<float> audio_buffer;
+
+  bool remuxing;
+};

system/loggerd/xattr_cache.py

@@ -0,0 +1,23 @@
+import errno
+
+import xattr
+
+_cached_attributes: dict[tuple, bytes | None] = {}
+
+def getxattr(path: str, attr_name: str) -> bytes | None:
+  key = (path, attr_name)
+  if key not in _cached_attributes:
+    try:
+      response = xattr.getxattr(path, attr_name)
+    except OSError as e:
+      # ENODATA (Linux) or ENOATTR (macOS) means attribute hasn't been set
+      if e.errno == errno.ENODATA or (hasattr(errno, 'ENOATTR') and e.errno == errno.ENOATTR):
+        response = None
+      else:
+        raise
+    _cached_attributes[key] = response
+  return _cached_attributes[key]
+
+def setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  _cached_attributes.pop((path, attr_name), None)
+  xattr.setxattr(path, attr_name, attr_value)

system/loggerd/zstd_writer.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <zstd.h>
+
+#include <string>
+#include <vector>
+#include <capnp/common.h>
+
+class ZstdFileWriter {
+public:
+  ZstdFileWriter(const std::string &filename, int compression_level);
+  ~ZstdFileWriter();
+  void write(void* data, size_t size);
+  inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); }
+
+private:
+  void flushCache(bool last_chunk);
+
+  size_t input_cache_capacity_ = 0;
+  std::vector<char> input_cache_;
+  std::vector<char> output_buffer_;
+  ZSTD_CStream *cstream_;
+  FILE* file_ = nullptr;
+};

system/logmessaged.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+import zmq
+from typing import NoReturn
+
+import cereal.messaging as messaging
+from openpilot.common.logging_extra import SwagLogFileFormatter
+from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import get_file_handler
+from openpilot.common.params import Params
+
+
+def main() -> NoReturn:
+  log_handler = get_file_handler()
+  log_handler.setFormatter(SwagLogFileFormatter(None))
+  log_level = 20  # logging.INFO
+
+  ctx = zmq.Context.instance()
+  sock = ctx.socket(zmq.PULL)
+  sock.bind(Paths.swaglog_ipc())
+
+  # and we publish them
+  log_message_sock = messaging.pub_sock('logMessage')
+  error_log_message_sock = messaging.pub_sock('errorLogMessage')
+
+  try:
+    while True:
+      dat = b''.join(sock.recv_multipart())
+      level = dat[0]
+      raw_bytes = dat[1:]
+
+      try:
+        record = dat[1:].decode("utf-8", errors="replace")
+      except Exception as e:
+        print(f"decode error: {e}, skipping log")
+        print(f"Raw bytes (hex): {raw_bytes.hex()[:200]}...")  # 앞부분만 출력
+        Params().put_bool("CarrotException", True)
+        continue
+
+      if level >= log_level:
+        log_handler.emit(record)
+
+      if len(record) > 2*1024*1024:
+        print("WARNING: log too big to publish", len(record))
+        print(record[:100])
+        continue
+
+      # then we publish them
+      msg = messaging.new_message(None, valid=True, logMessage=record)
+      log_message_sock.send(msg.to_bytes())
+
+      if level >= 40:  # logging.ERROR
+        msg = messaging.new_message(None, valid=True, errorLogMessage=record)
+        error_log_message_sock.send(msg.to_bytes())
+  finally:
+    sock.close()
+    ctx.term()
+
+    # can hit this if interrupted during a rollover
+    try:
+      log_handler.close()
+    except ValueError:
+      pass
+
+if __name__ == "__main__":
+  main()

system/manager/build.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+from pathlib import Path
+
+# NOTE: Do NOT import anything here that needs be built (e.g. params)
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.spinner import Spinner
+from openpilot.common.text_window import TextWindow
+from openpilot.system.hardware import HARDWARE, AGNOS
+from openpilot.common.swaglog import cloudlog, add_file_handler
+from openpilot.system.version import get_build_metadata
+
+MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
+CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
+
+TOTAL_SCONS_NODES = 3130
+MAX_BUILD_PROGRESS = 100
+
+def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
+  env = os.environ.copy()
+  env['SCONS_PROGRESS'] = "1"
+  nproc = os.cpu_count()
+  if nproc is None:
+    nproc = 2
+
+  extra_args = ["--minimal"] if minimal else []
+
+  if AGNOS:
+    HARDWARE.set_power_save(False)
+    os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores
+
+  # building with all cores can result in using too
+  # much memory, so retry with less parallelism
+  compile_output: list[bytes] = []
+  for n in (nproc, nproc/2, 1):
+    compile_output.clear()
+    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    assert scons.stderr is not None
+
+    # Read progress from stderr and update spinner
+    while scons.poll() is None:
+      try:
+        line = scons.stderr.readline()
+        if line is None:
+          continue
+        line = line.rstrip()
+
+        prefix = b'progress: '
+        if line.startswith(prefix):
+          i = int(line[len(prefix):])
+          spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
+        elif len(line):
+          compile_output.append(line)
+          print(line.decode('utf8', 'replace'))
+      except Exception:
+        pass
+
+    if scons.returncode == 0:
+      break
+
+  if scons.returncode != 0:
+    # Read remaining output
+    if scons.stderr is not None:
+      compile_output += scons.stderr.read().split(b'\n')
+
+    # Build failed log errors
+    error_s = b"\n".join(compile_output).decode('utf8', 'replace')
+    add_file_handler(cloudlog)
+    cloudlog.error("scons build failed\n" + error_s)
+
+    # Show TextWindow
+    spinner.close()
+    if not os.getenv("CI"):
+      with TextWindow("openpilot failed to build\n \n" + error_s) as t:
+        t.wait_for_exit()
+    exit(1)
+
+  # enforce max cache size
+  cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
+  cache_files.sort(key=lambda f: f.stat().st_mtime)
+  cache_size = sum(f.stat().st_size for f in cache_files)
+  for f in cache_files:
+    if cache_size < MAX_CACHE_SIZE:
+      break
+    cache_size -= f.stat().st_size
+    f.unlink()
+
+
+if __name__ == "__main__":
+  spinner = Spinner()
+  spinner.update_progress(0, 100)
+  build_metadata = get_build_metadata()
+  build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)

system/manager/helpers.py

@@ -0,0 +1,67 @@
+import errno
+import fcntl
+import os
+import sys
+import pathlib
+import shutil
+import signal
+import subprocess
+import tempfile
+import threading
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+
+def unblock_stdout() -> None:
+  # get a non-blocking stdout
+  child_pid, child_pty = os.forkpty()
+  if child_pid != 0:  # parent
+
+    # child is in its own process group, manually pass kill signals
+    signal.signal(signal.SIGINT, lambda signum, frame: os.kill(child_pid, signal.SIGINT))
+    signal.signal(signal.SIGTERM, lambda signum, frame: os.kill(child_pid, signal.SIGTERM))
+
+    fcntl.fcntl(sys.stdout, fcntl.F_SETFL, fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+    while True:
+      try:
+        dat = os.read(child_pty, 4096)
+      except OSError as e:
+        if e.errno == errno.EIO:
+          break
+        continue
+
+      if not dat:
+        break
+
+      try:
+        sys.stdout.write(dat.decode('utf8'))
+      except (OSError, UnicodeDecodeError):
+        pass
+
+    # os.wait() returns a tuple with the pid and a 16 bit value
+    # whose low byte is the signal number and whose high byte is the exit status
+    exit_status = os.wait()[1] >> 8
+    os._exit(exit_status)
+
+
+def write_onroad_params(started, params):
+  params.put_bool("IsOnroad", started)
+  params.put_bool("IsOffroad", not started)
+
+
+def save_bootlog():
+  # copy current params
+  tmp = tempfile.mkdtemp()
+  params_dirname = pathlib.Path(Params().get_param_path()).name
+  params_dir = os.path.join(tmp, params_dirname)
+  shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True)
+
+  def fn(tmpdir):
+    env = os.environ.copy()
+    env['PARAMS_COPY_PATH'] = tmpdir
+    subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), env=env)
+    shutil.rmtree(tmpdir)
+  t = threading.Thread(target=fn, args=(tmp, ))
+  t.daemon = True
+  t.start()

system/manager/manager.py

@@ -0,0 +1,450 @@
+#!/usr/bin/env python3
+import datetime
+import os
+import signal
+import sys
+import traceback
+
+from cereal import log
+import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
+from openpilot.common.params import Params, ParamKeyType
+from openpilot.common.text_window import TextWindow
+from openpilot.system.hardware import HARDWARE
+from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog
+from openpilot.system.manager.process import ensure_running
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.common.swaglog import cloudlog, add_file_handler
+from openpilot.system.version import get_build_metadata, terms_version, training_version
+from openpilot.system.hardware.hw import Paths
+
+def get_default_params():
+  default_params : list[tuple[str, str | bytes]] = [
+    # kans
+    ("LongPitch", "1"),
+    ("EVTable", "1"),
+    ("CompletedTrainingVersion", "0"),
+    ("DisengageOnAccelerator", "0"),
+    ("GsmMetered", "1"),
+    ("HasAcceptedTerms", "0"),
+    ("LanguageSetting", "main_en"),
+    ("OpenpilotEnabledToggle", "1"),
+    ("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)),
+    ("IsMetric", "1"),
+    ("RecordAudio", "1"),
+
+    ("SearchInput", "0"),
+    ("GMapKey", "0"),
+    ("MapboxStyle", "0"),
+
+
+    ("LongitudinalPersonalityMax", "3"),
+    ("ShowDebugUI", "0"),
+    ("ShowTpms", "1"),
+    ("ShowDateTime", "1"),
+    ("ShowPathEnd", "1"),
+    ("ShowCustomBrightness", "100"),
+    ("ShowLaneInfo", "1"),
+    ("ShowRadarInfo", "1"),
+    ("ShowDeviceState", "1"),
+    ("ShowRouteInfo", "1"),
+    ("ShowPathMode", "9"),
+    ("ShowPathColor", "13"),
+    ("ShowPathColorCruiseOff", "19"),
+    ("ShowPathModeLane", "14"),
+    ("ShowPathColorLane", "13"),
+    ("ShowPlotMode", "0"),
+    ("AutoCruiseControl", "0"),
+    ("CruiseEcoControl", "2"),
+    ("CarrotCruiseDecel", "-1"),
+    ("CarrotCruiseAtcDecel", "-1"),
+    ("CommaLongAcc", "0"),
+    ("AutoGasTokSpeed", "0"),
+    ("AutoGasSyncSpeed", "1"),
+    ("AutoEngage", "0"),
+    ("DisableMinSteerSpeed", "0"),
+    ("SoftHoldMode", "0"),
+
+    ("AutoSpeedUptoRoadSpeedLimit", "0"),
+    ("AutoRoadSpeedAdjust", "50"),
+    ("AutoCurveSpeedLowerLimit", "30"),
+    ("AutoCurveSpeedFactor", "120"),
+    ("AutoCurveSpeedAggressiveness", "100"),
+
+    ("AutoTurnControl", "0"),
+    ("AutoTurnControlSpeedTurn", "20"),
+    ("AutoTurnControlTurnEnd", "6"),
+    ("AutoTurnMapChange", "0"),
+
+    ("AutoNaviSpeedCtrlEnd", "7"),
+    ("AutoNaviSpeedCtrlMode", "2"),
+    ("AutoNaviSpeedBumpTime", "1"),
+    ("AutoNaviSpeedBumpSpeed", "35"),
+    ("AutoNaviSpeedSafetyFactor", "105"),
+    ("AutoNaviSpeedDecelRate", "120"),
+    ("AutoRoadSpeedLimitOffset", "-1"),
+    ("AutoNaviCountDownMode", "2"),
+    ("TurnSpeedControlMode", "1"),
+    ("CarrotSmartSpeedControl", "0"),
+    ("MapTurnSpeedFactor", "90"),
+    ("ModelTurnSpeedFactor", "0"),
+    ("StoppingAccel", "0"),
+    ("StopDistanceCarrot", "550"),
+    ("JLeadFactor3", "0"),
+    ("CruiseButtonMode", "0"),
+    ("CancelButtonMode", "0"),
+    ("LfaButtonMode", "0"),
+    ("CruiseButtonTest1", "8"),
+    ("CruiseButtonTest2", "30"),
+    ("CruiseButtonTest3", "1"),
+    ("CruiseSpeedUnit", "10"),
+    ("CruiseSpeedUnitBasic", "1"),
+    ("CruiseSpeed1", "30"),
+    ("CruiseSpeed2", "50"),
+    ("CruiseSpeed3", "80"),
+    ("CruiseSpeed4", "110"),
+    ("CruiseSpeed5", "130"),
+    ("PaddleMode", "0"),
+    ("MyDrivingMode", "3"),
+    ("MyDrivingModeAuto", "0"),
+    ("TrafficLightDetectMode", "2"),
+    ("CruiseMaxVals0", "160"),
+    ("CruiseMaxVals1", "200"),
+    ("CruiseMaxVals2", "160"),
+    ("CruiseMaxVals3", "130"),
+    ("CruiseMaxVals4", "110"),
+    ("CruiseMaxVals5", "95"),
+    ("CruiseMaxVals6", "80"),
+    ("LongTuningKpV", "100"),
+    ("LongTuningKiV", "0"),
+    ("LongTuningKf", "100"),
+    ("LongActuatorDelay", "20"),
+    ("VEgoStopping", "50"),
+    ("RadarReactionFactor", "100"),
+    ("EnableRadarTracks", "0"),
+    ("RadarLatFactor", "0"),
+    ("EnableCornerRadar", "0"),
+    ("HyundaiCameraSCC", "0"),
+    ("IsLdwsCar", "0"),
+    ("CanfdHDA2", "0"),
+    ("CanfdDebug", "0"),
+    ("SoundVolumeAdjust", "100"),
+    ("SoundVolumeAdjustEngage", "10"),
+    ("TFollowGap1", "110"),
+    ("TFollowGap2", "120"),
+    ("TFollowGap3", "140"),
+    ("TFollowGap4", "160"),
+    ("DynamicTFollow", "0"),
+    ("AChangeCostStarting", "10"),
+    ("TrafficStopDistanceAdjust", "400"),
+    ("DynamicTFollowLC", "100"),
+    ("HapticFeedbackWhenSpeedCamera", "0"),
+    ("UseLaneLineSpeed", "0"),
+    ("PathOffset", "0"),
+    ("UseLaneLineCurveSpeed", "0"),
+    ("AdjustLaneOffset", "0"),
+    ("LaneChangeNeedTorque", "0"),
+    ("LaneChangeDelay", "0"),
+    ("LaneChangeBsd", "0"),
+    ("MaxAngleFrames", "89"),
+    ("LateralTorqueCustom", "0"),
+    ("LateralTorqueAccelFactor", "2500"),
+    ("LateralTorqueFriction", "100"),
+    ("LateralTorqueKpV", "100"),
+    ("LateralTorqueKiV", "10"),
+    ("LateralTorqueKf", "100"),
+    ("LateralTorqueKd", "0"),
+    ("LatMpcPathCost", "200"),
+    ("LatMpcMotionCost", "7"),
+    ("LatMpcAccelCost", "120"),
+    ("LatMpcJerkCost", "4"),
+    ("LatMpcSteeringRateCost", "7"),
+    ("LatMpcInputOffset", "4"),
+    ("CustomSteerMax", "0"),
+    ("CustomSteerDeltaUp", "0"),
+    ("CustomSteerDeltaDown", "0"),
+    ("CustomSteerDeltaUpLC", "0"),
+    ("CustomSteerDeltaDownLC", "0"),
+    ("SpeedFromPCM", "2"),
+    ("SteerActuatorDelay", "0"),
+    ("LatSmoothSec", "13"),
+    ("MaxTimeOffroadMin", "60"),
+    ("DisableDM", "1"),
+    ("EnableConnect", "0"),
+    ("MuteDoor", "0"),
+    ("MuteSeatbelt", "0"),
+    ("RecordRoadCam", "0"),
+    ("HDPuse", "0"),
+    ("CruiseOnDist", "400"),
+    ("HotspotOnBoot", "0"),
+    ("SoftwareMenu", "1"),
+    ("CustomSR", "0"),
+    ("SteerRatioRate", "100"),
+    ("NNFF", "0"),
+    ("NNFFLite", "0"),
+
+    ("ForceOffroad", "0"),
+    ("BydModifiedStockLong", "1"),
+    ("AlwaysOnLKAS", "0"),
+    ("BydAutoTuning", "0"),
+    ("BydLatUseSiglin", "1"),
+    ("CameraOffset", "8"),
+    ("BydBsdType2", "0"),
+    ("UseRedPanda", "1"),
+    ("KeepLkasPassive", "0"),
+    ("UseSteerRateLimiter", "1"),
+    ("SteerRateLimLoSpd", "132"),
+    ("SteerRateLimHiSpd", "64"),
+    ("BydMpcTsr", "0"),
+    ("BydLowSpdLong", "1"),
+    ("SpeedCorrect30", "0"),
+    ("SpeedCorrect60", "0"),
+    ("SpeedCorrect90", "0"),
+    ("SpeedCorrect120", "0"),
+    ("LateralAngleSpdUp0", "500"),
+    ("LateralAngleSpdDn0", "500"),
+    ("LateralAngleSpdBp1", "30"),
+    ("LateralAngleSpdUp1", "80"),
+    ("LateralAngleSpdDn1", "350"),
+    ("LateralAngleSpdBp2", "70"),
+    ("LateralAngleSpdUp2", "15"),
+    ("LateralAngleSpdDn2", "40"),
+    ("LateralAngleTorqMax", "30"),
+    ("LateralAngleTorqCut", "10"),
+  ]
+  return default_params
+
+def set_default_params():
+  params = Params()
+  default_params = get_default_params()
+  try:
+    default_params.remove(("GMapKey", "0"))
+    default_params.remove(("CompletedTrainingVersion", "0"))
+    default_params.remove(("LanguageSetting", "main_en"))
+    default_params.remove(("GsmMetered", "1"))
+  except ValueError:
+    pass
+  for k, v in default_params:
+    params.put(k, v)
+    print(f"SetToDefault[{k}]={v}")
+
+def get_default_params_key():
+  default_params = get_default_params()
+  all_keys = [key for key, _ in default_params]
+  return all_keys
+
+def manager_init() -> None:
+  save_bootlog()
+
+  build_metadata = get_build_metadata()
+
+  params = Params()
+  params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
+  params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
+  params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
+  if build_metadata.release_channel:
+    params.clear_all(ParamKeyType.DEVELOPMENT_ONLY)
+
+  default_params = get_default_params()
+
+  if params.get_bool("RecordFrontLock"):
+    params.put_bool("RecordFront", True)
+
+  # set unset params
+  for k, v in default_params:
+    if params.get(k) is None:
+      params.put(k, v)
+
+  # Create folders needed for msgq
+  try:
+    os.mkdir(Paths.shm_path())
+  except FileExistsError:
+    pass
+  except PermissionError:
+    print(f"WARNING: failed to make {Paths.shm_path()}")
+
+  # set params
+  serial = HARDWARE.get_serial()
+  params.put("Version", build_metadata.openpilot.version)
+  params.put("TermsVersion", terms_version)
+  params.put("TrainingVersion", training_version)
+  params.put("GitCommit", build_metadata.openpilot.git_commit)
+  params.put("GitCommitDate", build_metadata.openpilot.git_commit_date)
+  params.put("GitBranch", build_metadata.channel)
+  params.put("GitRemote", build_metadata.openpilot.git_origin)
+  params.put_bool("IsTestedBranch", build_metadata.tested_channel)
+  params.put_bool("IsReleaseBranch", build_metadata.release_channel)
+  params.put("HardwareSerial", serial)
+
+  # set dongle id
+  reg_res = register(show_spinner=True)
+  if reg_res:
+    dongle_id = reg_res
+  else:
+    raise Exception(f"Registration failed for device {serial}")
+  os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
+  os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
+  os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
+  os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
+
+  if not build_metadata.openpilot.is_dirty:
+    os.environ['CLEAN'] = '1'
+
+  # init logging
+  sentry.init(sentry.SentryProject.SELFDRIVE)
+  cloudlog.bind_global(dongle_id=dongle_id,
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
+                       device=HARDWARE.get_device_type())
+
+  # preimport all processes
+  for p in managed_processes.values():
+    p.prepare()
+
+
+def manager_cleanup() -> None:
+  # send signals to kill all procs
+  for p in managed_processes.values():
+    p.stop(block=False)
+
+  # ensure all are killed
+  for p in managed_processes.values():
+    p.stop(block=True)
+
+  cloudlog.info("everything is dead")
+
+
+def manager_thread() -> None:
+  cloudlog.bind(daemon="manager")
+  cloudlog.info("manager start")
+  cloudlog.info({"environ": os.environ})
+
+  params = Params()
+
+  ignore: list[str] = []
+  if params.get("DongleId", encoding='utf8') in (None, UNREGISTERED_DONGLE_ID):
+    ignore += ["manage_athenad", "uploader"]
+  if os.getenv("NOBOARD") is not None:
+    ignore.append("pandad")
+  ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0]
+
+  if params.get("HardwareC3xLite"):
+    ignore += ["micd", "soundd", "loggerd"]
+    params.put("RecordAudio", "0")
+
+  sm = messaging.SubMaster(['deviceState', 'carParams'], poll='deviceState')
+  pm = messaging.PubMaster(['managerState'])
+
+  write_onroad_params(False, params)
+  ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore)
+
+  print_timer = 0
+
+  started_prev = False
+
+  while True:
+    sm.update(1000)
+
+    started = sm['deviceState'].started
+
+    if started and not started_prev:
+      params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
+    elif not started and started_prev:
+      params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
+
+    # update onroad params, which drives pandad's safety setter thread
+    if started != started_prev:
+      write_onroad_params(started, params)
+
+    started_prev = started
+
+    ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)
+
+    running = ' '.join("{}{}\u001b[0m".format("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
+                       for p in managed_processes.values() if p.proc)
+    print_timer = (print_timer + 1)%10
+    if print_timer == 0:
+      print(running)
+    cloudlog.debug(running)
+
+    # send managerState
+    msg = messaging.new_message('managerState', valid=True)
+    msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()]
+    pm.send('managerState', msg)
+
+    # Exit main loop when uninstall/shutdown/reboot is needed
+    shutdown = False
+    for param in ("DoUninstall", "DoShutdown", "DoReboot"):
+      if params.get_bool(param):
+        shutdown = True
+        params.put("LastManagerExitReason", f"{param} {datetime.datetime.now()}")
+        cloudlog.warning(f"Shutting down manager - {param} set")
+
+    if shutdown:
+      break
+
+def main() -> None:
+  manager_init()
+  print(f"python ../../opendbc/car/hyundai/values.py > {Params().get_param_path()}/SupportedCars")
+  os.system(f"python ../../opendbc/car/hyundai/values.py > {Params().get_param_path()}/SupportedCars")
+  os.system(f"python ../../opendbc/car/gm/values.py > {Params().get_param_path()}/SupportedCars_gm")
+  os.system(f"python ../../opendbc/car/toyota/values.py > {Params().get_param_path()}/SupportedCars_toyota")
+  os.system(f"python ../../opendbc/car/mazda/values.py > {Params().get_param_path()}/SupportedCars_mazda")
+  os.system(f"python ../../opendbc/car/byd/values.py > {Params().get_param_path()}/SupportedCars_byd")
+
+  if os.getenv("PREPAREONLY") is not None:
+    return
+
+  # SystemExit on sigterm
+  signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
+
+  try:
+    manager_thread()
+  except Exception:
+    traceback.print_exc()
+    sentry.capture_exception()
+  finally:
+    manager_cleanup()
+
+  params = Params()
+  if params.get_bool("DoUninstall"):
+    cloudlog.warning("uninstalling")
+    HARDWARE.uninstall()
+  elif params.get_bool("DoReboot"):
+    cloudlog.warning("reboot")
+    HARDWARE.reboot()
+  elif params.get_bool("DoShutdown"):
+    cloudlog.warning("shutdown")
+    HARDWARE.shutdown()
+
+
+if __name__ == "__main__":
+  unblock_stdout()
+
+  try:
+    main()
+  except KeyboardInterrupt:
+    print("got CTRL-C, exiting")
+  except Exception:
+    add_file_handler(cloudlog)
+    cloudlog.exception("Manager failed to start")
+
+    try:
+      managed_processes['ui'].stop()
+    except Exception:
+      pass
+
+    # Show last 3 lines of traceback
+    error = traceback.format_exc(-3)
+    error = "Manager failed to start\n\n" + error
+    with TextWindow(error) as t:
+      t.wait_for_exit()
+
+    raise
+
+  # manual exit because we are forked
+  sys.exit(0)

system/manager/process.py

@@ -0,0 +1,301 @@
+import importlib
+import os
+import signal
+import struct
+import time
+import subprocess
+from collections.abc import Callable, ValuesView
+from abc import ABC, abstractmethod
+from multiprocessing import Process
+
+from setproctitle import setproctitle
+
+from cereal import car, log
+import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware.hw import Paths
+
+WATCHDOG_FN = f"{Paths.shm_path()}/wd_"
+ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
+
+
+def launcher(proc: str, name: str) -> None:
+  try:
+    # import the process
+    mod = importlib.import_module(proc)
+
+    # rename the process
+    setproctitle(proc)
+
+    # create new context since we forked
+    messaging.reset_context()
+
+    # add daemon name tag to logs
+    cloudlog.bind(daemon=name)
+    sentry.set_tag("daemon", name)
+
+    # exec the process
+    mod.main()
+  except KeyboardInterrupt:
+    cloudlog.warning(f"child {proc} got SIGINT")
+  except Exception:
+    # can't install the crash handler because sys.excepthook doesn't play nice
+    # with threads, so catch it here.
+    sentry.capture_exception()
+    raise
+
+
+def nativelauncher(pargs: list[str], cwd: str, name: str) -> None:
+  os.environ['MANAGER_DAEMON'] = name
+
+  # exec the process
+  os.chdir(cwd)
+  os.execvp(pargs[0], pargs)
+
+
+def join_process(process: Process, timeout: float) -> None:
+  # Process().join(timeout) will hang due to a python 3 bug: https://bugs.python.org/issue28382
+  # We have to poll the exitcode instead
+  t = time.monotonic()
+  while time.monotonic() - t < timeout and process.exitcode is None:
+    time.sleep(0.001)
+
+
+class ManagerProcess(ABC):
+  daemon = False
+  sigkill = False
+  should_run: Callable[[bool, Params, car.CarParams], bool]
+  proc: Process | None = None
+  enabled = True
+  name = ""
+
+  last_watchdog_time = 0
+  watchdog_max_dt: int | None = None
+  watchdog_seen = False
+  shutting_down = False
+
+  @abstractmethod
+  def prepare(self) -> None:
+    pass
+
+  @abstractmethod
+  def start(self) -> None:
+    pass
+
+  def restart(self) -> None:
+    self.stop(sig=signal.SIGKILL)
+    self.start()
+
+  def check_watchdog(self, started: bool) -> None:
+    if self.watchdog_max_dt is None or self.proc is None:
+      return
+
+    try:
+      fn = WATCHDOG_FN + str(self.proc.pid)
+      with open(fn, "rb") as f:
+        # TODO: why can't pylint find struct.unpack?
+        self.last_watchdog_time = struct.unpack('Q', f.read())[0]
+    except Exception:
+      pass
+
+    dt = time.monotonic() - self.last_watchdog_time / 1e9
+
+    if dt > self.watchdog_max_dt:
+      if self.watchdog_seen and ENABLE_WATCHDOG:
+        cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})")
+        self.restart()
+    else:
+      self.watchdog_seen = True
+
+  def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals = None) -> int | None:
+    if self.proc is None:
+      return None
+
+    if self.proc.exitcode is None:
+      if not self.shutting_down:
+        cloudlog.info(f"killing {self.name}")
+        if sig is None:
+          sig = signal.SIGKILL if self.sigkill else signal.SIGINT
+        self.signal(sig)
+        self.shutting_down = True
+
+        if not block:
+          return None
+
+      join_process(self.proc, 5)
+
+      # If process failed to die send SIGKILL
+      if self.proc.exitcode is None and retry:
+        cloudlog.info(f"killing {self.name} with SIGKILL")
+        self.signal(signal.SIGKILL)
+        self.proc.join()
+
+    ret = self.proc.exitcode
+    cloudlog.info(f"{self.name} is dead with {ret}")
+
+    if self.proc.exitcode is not None:
+      self.shutting_down = False
+      self.proc = None
+
+    return ret
+
+  def signal(self, sig: int) -> None:
+    if self.proc is None:
+      return
+
+    # Don't signal if already exited
+    if self.proc.exitcode is not None and self.proc.pid is not None:
+      return
+
+    # Can't signal if we don't have a pid
+    if self.proc.pid is None:
+      return
+
+    cloudlog.info(f"sending signal {sig} to {self.name}")
+    os.kill(self.proc.pid, sig)
+
+  def get_process_state_msg(self):
+    state = log.ManagerState.ProcessState.new_message()
+    state.name = self.name
+    if self.proc:
+      state.running = self.proc.is_alive()
+      state.shouldBeRunning = self.proc is not None and not self.shutting_down
+      state.pid = self.proc.pid or 0
+      state.exitCode = self.proc.exitcode or 0
+    return state
+
+
+class NativeProcess(ManagerProcess):
+  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
+    self.name = name
+    self.cwd = cwd
+    self.cmdline = cmdline
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.watchdog_max_dt = watchdog_max_dt
+    self.launcher = nativelauncher
+
+  def prepare(self) -> None:
+    pass
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    cwd = os.path.join(BASEDIR, self.cwd)
+    cloudlog.info(f"starting process {self.name}")
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name))
+    self.proc.start()
+    self.watchdog_seen = False
+    self.shutting_down = False
+
+
+class PythonProcess(ManagerProcess):
+  def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
+    self.name = name
+    self.module = module
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.watchdog_max_dt = watchdog_max_dt
+    self.launcher = launcher
+
+  def prepare(self) -> None:
+    if self.enabled:
+      cloudlog.info(f"preimporting {self.module}")
+      try:
+        importlib.import_module(self.module)
+      except Exception as e:
+        print(f"failed to import {self.module}: {e}")
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    # TODO: this is just a workaround for this tinygrad check:
+    # https://github.com/tinygrad/tinygrad/blob/ac9c96dae1656dc220ee4acc39cef4dd449aa850/tinygrad/device.py#L26
+    name = self.name if "modeld" not in self.name else "MainProcess"
+
+    cloudlog.info(f"starting python {self.module}")
+    self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name))
+    self.proc.start()
+    self.watchdog_seen = False
+    self.shutting_down = False
+
+
+class DaemonProcess(ManagerProcess):
+  """Python process that has to stay running across manager restart.
+  This is used for athena so you don't lose SSH access when restarting manager."""
+  def __init__(self, name, module, param_name, enabled=True):
+    self.name = name
+    self.module = module
+    self.param_name = param_name
+    self.enabled = enabled
+    self.params = None
+
+  @staticmethod
+  def should_run(started, params, CP):
+    return True
+
+  def prepare(self) -> None:
+    pass
+
+  def start(self) -> None:
+    if self.params is None:
+      self.params = Params()
+
+    pid = self.params.get(self.param_name, encoding='utf-8')
+    if pid is not None:
+      try:
+        os.kill(int(pid), 0)
+        with open(f'/proc/{pid}/cmdline') as f:
+          if self.module in f.read():
+            # daemon is running
+            return
+      except (OSError, FileNotFoundError):
+        # process is dead
+        pass
+
+    cloudlog.info(f"starting daemon {self.name}")
+    proc = subprocess.Popen(['python', '-m', self.module],
+                               stdin=open('/dev/null'),
+                               stdout=open('/dev/null', 'w'),
+                               stderr=open('/dev/null', 'w'),
+                               preexec_fn=os.setpgrp)
+
+    self.params.put(self.param_name, str(proc.pid))
+
+  def stop(self, retry=True, block=True, sig=None) -> None:
+    pass
+
+
+def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None, CP: car.CarParams=None,
+                   not_run: list[str] | None=None) -> list[ManagerProcess]:
+  if not_run is None:
+    not_run = []
+
+  running = []
+  for p in procs:
+    if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
+      running.append(p)
+    else:
+      p.stop(block=False)
+
+    p.check_watchdog(started)
+
+  for p in running:
+    p.start()
+
+  return running