Release 260111

msgq/visionipc/__init__.py

@@ -0,0 +1,6 @@
+from msgq.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, VisionStreamType, get_endpoint_name
+assert VisionBuf
+assert VisionIpcClient
+assert VisionIpcServer
+assert VisionStreamType
+assert get_endpoint_name

msgq/visionipc/tests/test_visionipc.py

@@ -0,0 +1,99 @@
+import os
+import time
+import random
+import numpy as np
+from msgq.visionipc import VisionIpcServer, VisionIpcClient, VisionStreamType
+
+def zmq_sleep(t=1):
+  if "ZMQ" in os.environ:
+    time.sleep(t)
+
+
+class TestVisionIpc:
+
+  def setup_vipc(self, name, *stream_types, num_buffers=1, width=100, height=100, conflate=False):
+    self.server = VisionIpcServer(name)
+    for stream_type in stream_types:
+      self.server.create_buffers(stream_type, num_buffers, width, height)
+    self.server.start_listener()
+
+    if len(stream_types):
+      self.client = VisionIpcClient(name, stream_types[0], conflate)
+      assert self.client.connect(True)
+    else:
+      self.client = None
+
+    zmq_sleep()
+    return self.server, self.client
+
+  def test_connect(self):
+    self.setup_vipc("camerad", VisionStreamType.VISION_STREAM_ROAD)
+    assert self.client.is_connected
+    del self.client
+    del self.server
+
+  def test_available_streams(self):
+    for k in range(4):
+      stream_types = set(random.choices([x.value for x in VisionStreamType], k=k))
+      self.setup_vipc("camerad", *stream_types)
+      available_streams = VisionIpcClient.available_streams("camerad", True)
+      assert available_streams == stream_types
+      del self.client
+      del self.server
+
+  def test_buffers(self):
+    width, height, num_buffers = 100, 200, 5
+    self.setup_vipc("camerad", VisionStreamType.VISION_STREAM_ROAD, num_buffers=num_buffers, width=width, height=height)
+    assert self.client.width == width
+    assert self.client.height == height
+    assert self.client.buffer_len > 0
+    assert self.client.num_buffers == num_buffers
+    del self.client
+    del self.server
+
+  def test_send_single_buffer(self):
+    self.setup_vipc("camerad", VisionStreamType.VISION_STREAM_ROAD)
+
+    buf = np.zeros(self.client.buffer_len, dtype=np.uint8)
+    buf.view('<i4')[0] = 1234
+    self.server.send(VisionStreamType.VISION_STREAM_ROAD, buf, frame_id=1337)
+
+    recv_buf = self.client.recv()
+    assert recv_buf is not None
+    assert recv_buf.data.view('<i4')[0] == 1234
+    assert self.client.frame_id == 1337
+    del self.client
+    del self.server
+
+  def test_no_conflate(self):
+    self.setup_vipc("camerad", VisionStreamType.VISION_STREAM_ROAD)
+
+    buf = np.zeros(self.client.buffer_len, dtype=np.uint8)
+    self.server.send(VisionStreamType.VISION_STREAM_ROAD, buf, frame_id=1)
+    self.server.send(VisionStreamType.VISION_STREAM_ROAD, buf, frame_id=2)
+
+    recv_buf = self.client.recv()
+    assert recv_buf is not None
+    assert self.client.frame_id == 1
+
+    recv_buf = self.client.recv()
+    assert recv_buf is not None
+    assert self.client.frame_id == 2
+    del self.client
+    del self.server
+
+  def test_conflate(self):
+    self.setup_vipc("camerad", VisionStreamType.VISION_STREAM_ROAD, conflate=True)
+
+    buf = np.zeros(self.client.buffer_len, dtype=np.uint8)
+    self.server.send(VisionStreamType.VISION_STREAM_ROAD, buf, frame_id=1)
+    self.server.send(VisionStreamType.VISION_STREAM_ROAD, buf, frame_id=2)
+
+    recv_buf = self.client.recv()
+    assert recv_buf is not None
+    assert self.client.frame_id == 2
+
+    recv_buf = self.client.recv()
+    assert recv_buf is None
+    del self.client
+    del self.server

msgq/visionipc/visionbuf.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "msgq/visionipc/visionipc.h"
+
+#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#define VISIONBUF_SYNC_FROM_DEVICE 0
+#define VISIONBUF_SYNC_TO_DEVICE 1
+
+enum VisionStreamType {
+  VISION_STREAM_ROAD,
+  VISION_STREAM_DRIVER,
+  VISION_STREAM_WIDE_ROAD,
+
+  VISION_STREAM_MAP,
+  VISION_STREAM_MAX,
+};
+
+class VisionBuf {
+ public:
+  size_t len = 0;
+  size_t mmap_len = 0;
+  void * addr = nullptr;
+  uint64_t *frame_id;
+  int fd = 0;
+
+  size_t width = 0;
+  size_t height = 0;
+  size_t stride = 0;
+  size_t uv_offset = 0;
+
+  // YUV
+  uint8_t * y = nullptr;
+  uint8_t * uv = nullptr;
+
+  // Visionipc
+  uint64_t server_id = 0;
+  size_t idx = 0;
+  VisionStreamType type;
+
+  // OpenCL
+  cl_mem buf_cl = nullptr;
+  cl_command_queue copy_q = nullptr;
+
+  // ion
+  int handle = 0;
+
+  void allocate(size_t len);
+  void import();
+  void init_cl(cl_device_id device_id, cl_context ctx);
+  void init_yuv(size_t width, size_t height, size_t stride, size_t uv_offset);
+  int sync(int dir);
+  int free();
+
+  void set_frame_id(uint64_t id);
+  uint64_t get_frame_id();
+};

msgq/visionipc/visionipc.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+
+int ipc_connect(const char* socket_path);
+int ipc_bind(const char* socket_path);
+int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
+                          int *out_num_fds);
+
+constexpr int VISIONIPC_MAX_FDS = 128;
+
+struct VisionIpcBufExtra {
+  uint32_t frame_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+  bool valid;
+};
+
+struct VisionIpcPacket {
+  uint64_t server_id;
+  size_t idx;
+  struct VisionIpcBufExtra extra;
+};

msgq/visionipc/visionipc.pxd

@@ -0,0 +1,59 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.set cimport set
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool, int
+
+cdef extern from "msgq/visionipc/visionbuf.h":
+  struct _cl_device_id
+  struct _cl_context
+  struct _cl_mem
+
+  ctypedef _cl_device_id * cl_device_id
+  ctypedef _cl_context * cl_context
+  ctypedef _cl_mem * cl_mem
+
+  cdef enum VisionStreamType:
+    pass
+
+  cdef cppclass VisionBuf:
+    void * addr
+    size_t len
+    size_t width
+    size_t height
+    size_t stride
+    size_t uv_offset
+    cl_mem buf_cl
+    void set_frame_id(uint64_t id)
+
+cdef extern from "msgq/visionipc/visionipc.h":
+  struct VisionIpcBufExtra:
+    uint32_t frame_id
+    uint64_t timestamp_sof
+    uint64_t timestamp_eof
+    bool valid
+
+cdef extern from "msgq/visionipc/visionipc_server.h":
+  string get_endpoint_name(string, VisionStreamType)
+
+  cdef cppclass VisionIpcServer:
+    VisionIpcServer(string, void*, void*)
+    void create_buffers(VisionStreamType, size_t, size_t, size_t)
+    void create_buffers_with_sizes(VisionStreamType, size_t, size_t, size_t, size_t, size_t, size_t)
+    VisionBuf * get_buffer(VisionStreamType)
+    void send(VisionBuf *, VisionIpcBufExtra *, bool)
+    void start_listener()
+
+cdef extern from "msgq/visionipc/visionipc_client.h":
+  cdef cppclass VisionIpcClient:
+    int num_buffers
+    VisionBuf buffers[1]
+    VisionIpcClient(string, VisionStreamType, bool, void*, void*)
+    VisionBuf * recv(VisionIpcBufExtra *, int)
+    bool connect(bool)
+    bool is_connected()
+    @staticmethod
+    set[VisionStreamType] getAvailableStreams(string, bool)

msgq/visionipc/visionipc_client.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <set>
+#include <string>
+
+#include "msgq/ipc.h"
+#include "msgq/visionipc/visionbuf.h"
+
+
+class VisionIpcClient {
+private:
+  std::string name;
+  Context * msg_ctx;
+  SubSocket * sock;
+  Poller * poller;
+
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+
+public:
+  bool connected = false;
+  VisionStreamType type;
+  int num_buffers = 0;
+  VisionBuf buffers[VISIONIPC_MAX_FDS];
+  VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcClient();
+  VisionBuf * recv(VisionIpcBufExtra * extra=nullptr, const int timeout_ms=100);
+  bool connect(bool blocking=true);
+  bool is_connected() { return connected; }
+  static std::set<VisionStreamType> getAvailableStreams(const std::string &name, bool blocking = true);
+};

msgq/visionipc/visionipc_pyx.pxd

@@ -0,0 +1,15 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport cl_device_id, cl_context
+
+cdef class CLContext:
+  cdef cl_device_id device_id
+  cdef cl_context context
+
+cdef class VisionBuf:
+  cdef cppVisionBuf * buf
+
+  @staticmethod
+  cdef create(cppVisionBuf*)

msgq/visionipc/visionipc_pyx.pyx

@@ -0,0 +1,160 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+import numpy as np
+cimport numpy as cnp
+from cython.view cimport array
+from libc.string cimport memcpy
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool
+from libcpp.string cimport string
+
+from .visionipc cimport VisionIpcServer as cppVisionIpcServer
+from .visionipc cimport VisionIpcClient as cppVisionIpcClient
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport VisionIpcBufExtra
+from .visionipc cimport get_endpoint_name as cpp_get_endpoint_name
+
+
+def get_endpoint_name(string name, VisionStreamType stream):
+  return cpp_get_endpoint_name(name, stream).decode('utf-8')
+
+
+cpdef enum VisionStreamType:
+  VISION_STREAM_ROAD
+  VISION_STREAM_DRIVER
+  VISION_STREAM_WIDE_ROAD
+  VISION_STREAM_MAP
+
+
+cdef class VisionBuf:
+  @staticmethod
+  cdef create(cppVisionBuf * cbuf):
+    buf = VisionBuf()
+    buf.buf = cbuf
+    return buf
+
+  @property
+  def data(self):
+    return np.asarray(<cnp.uint8_t[:self.buf.len]> self.buf.addr)
+
+  @property
+  def width(self):
+    return self.buf.width
+
+  @property
+  def height(self):
+    return self.buf.height
+
+  @property
+  def stride(self):
+    return self.buf.stride
+
+  @property
+  def uv_offset(self):
+    return self.buf.uv_offset
+
+
+cdef class VisionIpcServer:
+  cdef cppVisionIpcServer * server
+
+  def __init__(self, string name):
+    self.server = new cppVisionIpcServer(name, NULL, NULL)
+
+  def create_buffers(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height):
+    self.server.create_buffers(tp, num_buffers, width, height)
+
+  def create_buffers_with_sizes(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset):
+    self.server.create_buffers_with_sizes(tp, num_buffers, width, height, size, stride, uv_offset)
+
+  def send(self, VisionStreamType tp, const unsigned char[:] data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
+    cdef cppVisionBuf * buf = self.server.get_buffer(tp)
+
+    # Populate buffer
+    assert buf.len == len(data)
+    memcpy(buf.addr, &data[0], len(data))
+    buf.set_frame_id(frame_id)
+
+    cdef VisionIpcBufExtra extra
+    extra.frame_id = frame_id
+    extra.timestamp_sof = timestamp_sof
+    extra.timestamp_eof = timestamp_eof
+
+    self.server.send(buf, &extra, False)
+
+  def start_listener(self):
+    self.server.start_listener()
+
+  def __dealloc__(self):
+    del self.server
+
+
+cdef class VisionIpcClient:
+  cdef cppVisionIpcClient * client
+  cdef VisionIpcBufExtra extra
+
+  def __cinit__(self, string name, VisionStreamType stream, bool conflate, CLContext context = None):
+    if context:
+      self.client = new cppVisionIpcClient(name, stream, conflate, context.device_id, context.context)
+    else:
+      self.client = new cppVisionIpcClient(name, stream, conflate, NULL, NULL)
+
+  def __dealloc__(self):
+    del self.client
+
+  @property
+  def width(self):
+    return self.client.buffers[0].width if self.client.num_buffers else None
+
+  @property
+  def height(self):
+    return self.client.buffers[0].height if self.client.num_buffers else None
+
+  @property
+  def stride(self):
+    return self.client.buffers[0].stride if self.client.num_buffers else None
+
+  @property
+  def uv_offset(self):
+    return self.client.buffers[0].uv_offset if self.client.num_buffers else None
+
+  @property
+  def buffer_len(self):
+    return self.client.buffers[0].len if self.client.num_buffers else None
+
+  @property
+  def num_buffers(self):
+    return self.client.num_buffers
+
+  @property
+  def frame_id(self):
+    return self.extra.frame_id
+
+  @property
+  def timestamp_sof(self):
+    return self.extra.timestamp_sof
+
+  @property
+  def timestamp_eof(self):
+    return self.extra.timestamp_eof
+
+  @property
+  def valid(self):
+    return self.extra.valid
+
+  def recv(self, int timeout_ms=100):
+    buf = self.client.recv(&self.extra, timeout_ms)
+    if not buf:
+      return None
+    return VisionBuf.create(buf)
+
+  def connect(self, bool blocking):
+    return self.client.connect(blocking)
+
+  def is_connected(self):
+    return self.client.is_connected()
+
+  @staticmethod
+  def available_streams(string name, bool block):
+    return cppVisionIpcClient.getAvailableStreams(name, block)

msgq/visionipc/visionipc_server.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <vector>
+#include <string>
+#include <thread>
+#include <atomic>
+#include <map>
+
+#include "msgq/ipc.h"
+#include "msgq/visionipc/visionbuf.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type);
+std::string get_ipc_path(const std::string &name);
+
+class VisionIpcServer {
+ private:
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+  uint64_t server_id;
+
+  std::atomic<bool> should_exit = false;
+  std::string name;
+  std::thread listener_thread;
+
+  std::map<VisionStreamType, std::atomic<size_t> > cur_idx;
+  std::map<VisionStreamType, std::vector<VisionBuf*> > buffers;
+
+  Context * msg_ctx;
+  std::map<VisionStreamType, PubSocket*> sockets;
+
+  void listener(void);
+
+ public:
+  VisionIpcServer(std::string name, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcServer();
+
+  VisionBuf * get_buffer(VisionStreamType type, int idx = -1);
+
+  void create_buffers(VisionStreamType type, size_t num_buffers, size_t width, size_t height);
+  void create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset);
+  void send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync=true);
+  void start_listener();
+};