Release 260111

tools/lib/tests/test_caching.py

@@ -0,0 +1,130 @@
+import http.server
+import os
+import shutil
+import socket
+import pytest
+
+from openpilot.selfdrive.test.helpers import http_server_context
+from openpilot.system.hardware.hw import Paths
+from openpilot.tools.lib.url_file import URLFile
+
+
+class CachingTestRequestHandler(http.server.BaseHTTPRequestHandler):
+  FILE_EXISTS = True
+
+  def do_GET(self):
+    if self.FILE_EXISTS:
+      self.send_response(206 if "Range" in self.headers else 200, b'1234')
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+  def do_HEAD(self):
+    if self.FILE_EXISTS:
+      self.send_response(200)
+      self.send_header("Content-Length", "4")
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+
+@pytest.fixture
+def host():
+  with http_server_context(handler=CachingTestRequestHandler) as (host, port):
+    yield f"http://{host}:{port}"
+
+class TestFileDownload:
+
+  def test_pipeline_defaults(self, host):
+    # TODO: parameterize the defaults so we don't rely on hard-coded values in xx
+
+    assert URLFile.pool_manager().pools._maxsize == 10# PoolManager num_pools param
+    pool_manager_defaults = {
+      "maxsize": 100,
+      "socket_options": [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),],
+    }
+    for k, v in pool_manager_defaults.items():
+      assert URLFile.pool_manager().connection_pool_kw.get(k) == v
+
+    retry_defaults = {
+      "total": 5,
+      "backoff_factor": 0.5,
+      "status_forcelist": [409, 429, 503, 504],
+    }
+    for k, v in retry_defaults.items():
+      assert getattr(URLFile.pool_manager().connection_pool_kw["retries"], k) == v
+
+    # ensure caching off by default and cache dir doesn't get created
+    os.environ.pop("FILEREADER_CACHE", None)
+    if os.path.exists(Paths.download_cache_root()):
+      shutil.rmtree(Paths.download_cache_root())
+    URLFile(f"{host}/test.txt").get_length()
+    URLFile(f"{host}/test.txt").read()
+    assert not os.path.exists(Paths.download_cache_root())
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    assert file_cached.get_length() == file_downloaded.get_length()
+    assert length + start if length is not None else 0 <= file_downloaded.get_length()
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    assert response_cached == response_downloaded
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    assert file_cached.get_length() == file_downloaded.get_length()
+    assert response_cached == response_downloaded
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ["FILEREADER_CACHE"] = "0"
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+  @pytest.mark.parametrize("cache_enabled", [True, False])
+  def test_recover_from_missing_file(self, host, cache_enabled):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+
+    file_url = f"{host}/test.png"
+
+    CachingTestRequestHandler.FILE_EXISTS = False
+    length = URLFile(file_url).get_length()
+    assert length == -1
+
+    CachingTestRequestHandler.FILE_EXISTS = True
+    length = URLFile(file_url).get_length()
+    assert length == 4

tools/lib/tests/test_comma_car_segments.py

@@ -0,0 +1,34 @@
+import pytest
+import requests
+from opendbc.car.fingerprints import MIGRATION
+from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database, get_url
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.route import SegmentRange
+
+
+@pytest.mark.skip(reason="huggingface is flaky, run this test manually to check for issues")
+class TestCommaCarSegments:
+  def test_database(self):
+    database = get_comma_car_segments_database()
+
+    platforms = database.keys()
+
+    assert len(platforms) > 100
+
+  def test_download_segment(self):
+    database = get_comma_car_segments_database()
+
+    fp = "SUBARU_FORESTER"
+
+    segment = database[fp][0]
+
+    sr = SegmentRange(segment)
+
+    url = get_url(sr.route_name, sr.slice)
+
+    resp = requests.get(url)
+    assert resp.status_code == 200
+
+    lr = LogReader(url)
+    CP = lr.first("carParams")
+    assert MIGRATION.get(CP.carFingerprint, CP.carFingerprint) == fp

tools/lib/tests/test_logreader.py

@@ -0,0 +1,256 @@
+import capnp
+import contextlib
+import io
+import shutil
+import tempfile
+import os
+import pytest
+import requests
+
+from parameterized import parameterized
+
+from cereal import log as capnp_log
+from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
+from openpilot.tools.lib.route import SegmentRange
+from openpilot.tools.lib.url_file import URLFileException
+
+NUM_SEGS = 17  # number of segments in the test route
+ALL_SEGS = list(range(NUM_SEGS))
+TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12"
+QLOG_FILE = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+
+
+def noop(segment: LogIterable):
+  return segment
+
+
+@contextlib.contextmanager
+def setup_source_scenario(mocker, is_internal=False):
+  internal_source_mock = mocker.patch("openpilot.tools.lib.logreader.internal_source")
+  internal_source_mock.__name__ = internal_source_mock._mock_name
+
+  openpilotci_source_mock = mocker.patch("openpilot.tools.lib.logreader.openpilotci_source")
+  openpilotci_source_mock.__name__ = openpilotci_source_mock._mock_name
+
+  comma_api_source_mock = mocker.patch("openpilot.tools.lib.logreader.comma_api_source")
+  comma_api_source_mock.__name__ = comma_api_source_mock._mock_name
+
+  if is_internal:
+    internal_source_mock.return_value = [QLOG_FILE]
+  else:
+    internal_source_mock.side_effect = InternalUnavailableException
+
+  openpilotci_source_mock.return_value = [None]
+  comma_api_source_mock.return_value = [QLOG_FILE]
+
+  yield
+
+
+class TestLogReader:
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", ALL_SEGS),
+    (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"{TEST_ROUTE}--0", [0]),
+    (f"{TEST_ROUTE}--5", [5]),
+    (f"{TEST_ROUTE}/0", [0]),
+    (f"{TEST_ROUTE}/5", [5]),
+    (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]),
+    (f"{TEST_ROUTE}/0:0", []),
+    (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]),
+    (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]),
+    (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]),
+    (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]),
+    (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]),
+    (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]),
+    (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]),
+    (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]),
+    (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]),
+    (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]),
+    (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '%7C')}", ALL_SEGS),
+  ])
+  def test_indirect_parsing(self, identifier, expected):
+    parsed = parse_indirect(identifier)
+    sr = SegmentRange(parsed)
+    assert list(sr.seg_idxs) == expected, identifier
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE.replace('/', '|')}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE}--5", f"{TEST_ROUTE}/5"),
+    (f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"),
+    (f"{TEST_ROUTE}/5:6/r", f"{TEST_ROUTE}/5:6/r"),
+    (f"{TEST_ROUTE}/5", f"{TEST_ROUTE}/5"),
+  ])
+  def test_canonical_name(self, identifier, expected):
+    sr = SegmentRange(identifier)
+    assert str(sr) == expected
+
+  @pytest.mark.parametrize("cache_enabled", [True, False])
+  def test_direct_parsing(self, mocker, cache_enabled):
+    file_exists_mock = mocker.patch("openpilot.tools.lib.logreader.file_exists")
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+    qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+
+    with requests.get(QLOG_FILE, stream=True) as r:
+      with qlog as f:
+        shutil.copyfileobj(r.raw, f)
+
+    for f in [QLOG_FILE, qlog.name]:
+      l = len(list(LogReader(f)))
+      assert l > 100
+
+    with pytest.raises(URLFileException) if not cache_enabled else pytest.raises(AssertionError):
+      l = len(list(LogReader(QLOG_FILE.replace("/3/", "/200/"))))
+
+    # file_exists should not be called for direct files
+    assert file_exists_mock.call_count == 0
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}///",),
+    (f"{TEST_ROUTE}---",),
+    (f"{TEST_ROUTE}/-4:--2",),
+    (f"{TEST_ROUTE}/-a",),
+    (f"{TEST_ROUTE}/j",),
+    (f"{TEST_ROUTE}/0:1:2:3",),
+    (f"{TEST_ROUTE}/:::3",),
+    (f"{TEST_ROUTE}3",),
+    (f"{TEST_ROUTE}-3",),
+    (f"{TEST_ROUTE}--3a",),
+  ])
+  def test_bad_ranges(self, segment_range):
+    with pytest.raises(AssertionError):
+      _ = SegmentRange(segment_range).seg_idxs
+
+  @pytest.mark.parametrize("segment_range, api_call", [
+    (f"{TEST_ROUTE}/0", False),
+    (f"{TEST_ROUTE}/:2", False),
+    (f"{TEST_ROUTE}/0:", True),
+    (f"{TEST_ROUTE}/-1", True),
+    (f"{TEST_ROUTE}", True),
+  ])
+  def test_slicing_api_call(self, mocker, segment_range, api_call):
+    max_seg_mock = mocker.patch("openpilot.tools.lib.route.get_max_seg_number_cached")
+    max_seg_mock.return_value = NUM_SEGS
+    _ = SegmentRange(segment_range).seg_idxs
+    assert api_call == max_seg_mock.called
+
+  @pytest.mark.slow
+  def test_modes(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.QLOG)))
+    rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.RLOG)))
+
+    assert qlog_len * 6 < rlog_len
+
+  @pytest.mark.slow
+  def test_modes_from_name(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q")))
+    rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/r")))
+
+    assert qlog_len * 6 < rlog_len
+
+  @pytest.mark.slow
+  def test_list(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q")))
+    qlog_len_2 = len(list(LogReader([f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"])))
+
+    assert qlog_len * 2 == qlog_len_2
+
+  @pytest.mark.slow
+  def test_multiple_iterations(self, mocker):
+    init_mock = mocker.patch("openpilot.tools.lib.logreader._LogFileReader")
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    qlog_len1 = len(list(lr))
+    qlog_len2 = len(list(lr))
+
+    # ensure we don't create multiple instances of _LogFileReader, which means downloading the files twice
+    assert init_mock.call_count == 1
+
+    assert qlog_len1 == qlog_len2
+
+  @pytest.mark.slow
+  def test_helpers(self):
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    assert lr.first("carParams").carFingerprint == "SUBARU OUTBACK 6TH GEN"
+    assert 0 < len(list(lr.filter("carParams"))) < len(list(lr))
+
+  @parameterized.expand([(True,), (False,)])
+  @pytest.mark.slow
+  def test_run_across_segments(self, cache_enabled):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+    lr = LogReader(f"{TEST_ROUTE}/0:4")
+    assert len(lr.run_across_segments(4, noop)) == len(list(lr))
+
+  @pytest.mark.slow
+  def test_auto_mode(self, subtests, mocker):
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    qlog_len = len(list(lr))
+    log_paths_mock = mocker.patch("openpilot.tools.lib.route.Route.log_paths")
+    log_paths_mock.return_value = [None] * NUM_SEGS
+    # Should fall back to qlogs since rlogs are not available
+
+    with subtests.test("interactive_yes"):
+      mocker.patch("sys.stdin", new=io.StringIO("y\n"))
+      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source)
+      log_len = len(list(lr))
+      assert qlog_len == log_len
+
+    with subtests.test("interactive_no"):
+      mocker.patch("sys.stdin", new=io.StringIO("n\n"))
+      with pytest.raises(AssertionError):
+        lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source)
+
+    with subtests.test("non_interactive"):
+      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, source=comma_api_source)
+      log_len = len(list(lr))
+      assert qlog_len == log_len
+
+  @pytest.mark.parametrize("is_internal", [True, False])
+  @pytest.mark.slow
+  def test_auto_source_scenarios(self, mocker, is_internal):
+    lr = LogReader(QLOG_FILE)
+    qlog_len = len(list(lr))
+
+    with setup_source_scenario(mocker, is_internal=is_internal):
+      lr = LogReader(f"{TEST_ROUTE}/0/q")
+      log_len = len(list(lr))
+      assert qlog_len == log_len
+
+  @pytest.mark.slow
+  def test_sort_by_time(self):
+    msgs = list(LogReader(f"{TEST_ROUTE}/0/q"))
+    assert msgs != sorted(msgs, key=lambda m: m.logMonoTime)
+
+    msgs = list(LogReader(f"{TEST_ROUTE}/0/q", sort_by_time=True))
+    assert msgs == sorted(msgs, key=lambda m: m.logMonoTime)
+
+  def test_only_union_types(self):
+    with tempfile.NamedTemporaryFile() as qlog:
+      # write valid Event messages
+      num_msgs = 100
+      with open(qlog.name, "wb") as f:
+        f.write(b"".join(capnp_log.Event.new_message().to_bytes() for _ in range(num_msgs)))
+
+      msgs = list(LogReader(qlog.name))
+      assert len(msgs) == num_msgs
+      [m.which() for m in msgs]
+
+      # append non-union Event message
+      event_msg = capnp_log.Event.new_message()
+      non_union_bytes = bytearray(event_msg.to_bytes())
+      non_union_bytes[event_msg.total_size.word_count * 8] = 0xff  # set discriminant value out of range using Event word offset
+      with open(qlog.name, "ab") as f:
+        f.write(non_union_bytes)
+
+      # ensure new message is added, but is not a union type
+      msgs = list(LogReader(qlog.name))
+      assert len(msgs) == num_msgs + 1
+      with pytest.raises(capnp.KjException):
+        [m.which() for m in msgs]
+
+      # should not be added when only_union_types=True
+      msgs = list(LogReader(qlog.name, only_union_types=True))
+      assert len(msgs) == num_msgs
+      [m.which() for m in msgs]

tools/lib/tests/test_readers.py

@@ -0,0 +1,63 @@
+import pytest
+import requests
+import tempfile
+
+from collections import defaultdict
+import numpy as np
+from openpilot.tools.lib.framereader import FrameReader
+from openpilot.tools.lib.logreader import LogReader
+
+
+class TestReaders:
+  @pytest.mark.skip("skip for bandwidth reasons")
+  def test_logreader(self):
+    def _check_data(lr):
+      hist = defaultdict(int)
+      for l in lr:
+        hist[l.which()] += 1
+
+      assert hist['carControl'] == 6000
+      assert hist['logMessage'] == 6857
+
+    with tempfile.NamedTemporaryFile(suffix=".bz2") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      lr_file = LogReader(fp.name)
+      _check_data(lr_file)
+
+    lr_url = LogReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
+    _check_data(lr_url)
+
+  @pytest.mark.skip("skip for bandwidth reasons")
+  def test_framereader(self):
+    def _check_data(f):
+      assert f.frame_count == 1200
+      assert f.w == 1164
+      assert f.h == 874
+
+      frame_first_30 = f.get(0, 30)
+      assert len(frame_first_30) == 30
+
+      print(frame_first_30[15])
+
+      print("frame_0")
+      frame_0 = f.get(0, 1)
+      frame_15 = f.get(15, 1)
+
+      print(frame_15[0])
+
+      assert np.all(frame_first_30[0] == frame_0[0])
+      assert np.all(frame_first_30[15] == frame_15[0])
+
+    with tempfile.NamedTemporaryFile(suffix=".hevc") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      fr_file = FrameReader(fp.name)
+      _check_data(fr_file)
+
+    fr_url = FrameReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true")
+    _check_data(fr_url)

tools/lib/tests/test_route_library.py

@@ -0,0 +1,27 @@
+from collections import namedtuple
+
+from openpilot.tools.lib.route import SegmentName
+
+class TestRouteLibrary:
+  def test_segment_name_formats(self):
+    Case = namedtuple('Case', ['input', 'expected_route', 'expected_segment_num', 'expected_data_dir'])
+
+    cases = [ Case("a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, None),
+              Case("a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19/3", "a2a0ccea32023010|2023-07-27--13-01-19", 3, None),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, "/data/media/0/realdata") ]
+
+    def _validate(case):
+      route_or_segment_name = case.input
+
+      s = SegmentName(route_or_segment_name, allow_route_name=True)
+
+      assert str(s.route_name) == case.expected_route
+      assert s.segment_num == case.expected_segment_num
+      assert s.data_dir == case.expected_data_dir
+
+    for case in cases:
+      _validate(case)