Release 260111

tools/webcam/README.md

@@ -0,0 +1,35 @@
+# Run openpilot with webcam on PC
+
+What's needed:
+- Ubuntu 24.04 ([WSL2 is not supported](https://github.com/commaai/openpilot/issues/34216))
+- GPU (recommended)
+- Two USB webcams, at least 720p and 78 degrees FOV (e.g. Logitech C920/C615)
+- [Car harness](https://comma.ai/shop/products/comma-car-harness) with black panda to connect to your car
+- [Panda paw](https://comma.ai/shop/products/panda-paw) or USB-A to USB-A cable to connect panda to your computer
+That's it!
+
+## Setup openpilot
+- Follow [this readme](../README.md) to install and build the requirements
+- Install OpenCL Driver
+```
+sudo apt install pocl-opencl-icd
+```
+
+## Connect the hardware
+- Connect the road facing camera first, then the driver facing camera
+- Connect your computer to panda
+
+## GO
+```
+USE_WEBCAM=1 system/manager/manager.py
+```
+- Start the car, then the UI should show the road webcam's view
+- Adjust and secure the webcams.
+- Finish calibration and engage!
+
+## Specify Cameras
+
+Use the `ROAD_CAM`, `DRIVER_CAM`, and optional `WIDE_CAM` environment variables to specify which camera is which (ie. `DRIVER_CAM=2` uses `/dev/video2` for the driver-facing camera):
+```
+USE_WEBCAM=1 ROAD_CAM=4 WIDE_CAM=6 system/manager/manager.py
+```

tools/webcam/camera.py

@@ -0,0 +1,29 @@
+import av
+
+class Camera:
+  def __init__(self, cam_type_state, stream_type, camera_id):
+    try:
+      camera_id = int(camera_id)
+    except ValueError: # allow strings, ex: /dev/video0
+      pass
+    self.cam_type_state = cam_type_state
+    self.stream_type = stream_type
+    self.cur_frame_id = 0
+
+    self.container = av.open(camera_id)
+    assert self.container.streams.video, f"Can't open video stream for camera {camera_id}"
+    self.video_stream = self.container.streams.video[0]
+    self.W = self.video_stream.codec_context.width
+    self.H = self.video_stream.codec_context.height
+
+  @classmethod
+  def bgr2nv12(self, bgr):
+    frame = av.VideoFrame.from_ndarray(bgr, format='bgr24')
+    return frame.reformat(format='nv12').to_ndarray()
+
+  def read_frames(self):
+    for frame in self.container.decode(self.video_stream):
+      img = frame.to_rgb().to_ndarray()[:,:, ::-1] # convert to bgr24
+      yuv = Camera.bgr2nv12(img)
+      yield yuv.data.tobytes()
+    self.container.close()

tools/webcam/camerad.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import threading
+import os
+from collections import namedtuple
+
+from msgq.visionipc import VisionIpcServer, VisionStreamType
+from cereal import messaging
+
+from openpilot.tools.webcam.camera import Camera
+from openpilot.common.realtime import Ratekeeper
+
+WIDE_CAM = os.getenv("WIDE_CAM")
+CameraType = namedtuple("CameraType", ["msg_name", "stream_type", "cam_id"])
+CAMERAS = [
+  CameraType("roadCameraState", VisionStreamType.VISION_STREAM_ROAD, os.getenv("ROAD_CAM", "0")),
+  CameraType("driverCameraState", VisionStreamType.VISION_STREAM_DRIVER, os.getenv("DRIVER_CAM", "2")),
+]
+if WIDE_CAM:
+  CAMERAS.append(CameraType("wideRoadCameraState", VisionStreamType.VISION_STREAM_WIDE_ROAD, WIDE_CAM))
+
+class Camerad:
+  def __init__(self):
+    self.pm = messaging.PubMaster([c.msg_name for c in CAMERAS])
+    self.vipc_server = VisionIpcServer("camerad")
+
+    self.cameras = []
+    for c in CAMERAS:
+      cam_device = f"/dev/video{c.cam_id}"
+      print(f"opening {c.msg_name} at {cam_device}")
+      cam = Camera(c.msg_name, c.stream_type, cam_device)
+      self.cameras.append(cam)
+      self.vipc_server.create_buffers(c.stream_type, 20, cam.W, cam.H)
+
+    self.vipc_server.start_listener()
+
+  def _send_yuv(self, yuv, frame_id, pub_type, yuv_type):
+    eof = int(frame_id * 0.05 * 1e9)
+    self.vipc_server.send(yuv_type, yuv, frame_id, eof, eof)
+    dat = messaging.new_message(pub_type, valid=True)
+    msg = {
+      "frameId": frame_id,
+      "transform": [1.0, 0.0, 0.0,
+                    0.0, 1.0, 0.0,
+                    0.0, 0.0, 1.0]
+    }
+    setattr(dat, pub_type, msg)
+    self.pm.send(pub_type, dat)
+
+  def camera_runner(self, cam):
+    rk = Ratekeeper(20, None)
+    for yuv in cam.read_frames():
+      self._send_yuv(yuv, cam.cur_frame_id, cam.cam_type_state, cam.stream_type)
+      cam.cur_frame_id += 1
+      rk.keep_time()
+
+  def run(self):
+    threads = []
+    for cam in self.cameras:
+      cam_thread = threading.Thread(target=self.camera_runner, args=(cam,))
+      cam_thread.start()
+      threads.append(cam_thread)
+
+    for t in threads:
+      t.join()
+
+
+def main():
+  camerad = Camerad()
+  camerad.run()
+
+
+if __name__ == "__main__":
+  main()