1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
7 #include <linux/mutex.h>
8 #include <linux/completion.h>
9 #include <linux/kthread.h>
10 #include <linux/kref.h>
11 #include <linux/rcupdate.h>
12 #include <linux/wait.h>
14 #include "vchiq_cfg.h"
18 /* Run time control of log level, based on KERN_XXX level. */
19 #define VCHIQ_LOG_DEFAULT 4
20 #define VCHIQ_LOG_ERROR 3
21 #define VCHIQ_LOG_WARNING 4
22 #define VCHIQ_LOG_INFO 6
23 #define VCHIQ_LOG_TRACE 7
25 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: "
27 #ifndef vchiq_log_error
28 #define vchiq_log_error(cat, fmt, ...) \
29 do { if (cat >= VCHIQ_LOG_ERROR) \
30 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
32 #ifndef vchiq_log_warning
33 #define vchiq_log_warning(cat, fmt, ...) \
34 do { if (cat >= VCHIQ_LOG_WARNING) \
35 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
37 #ifndef vchiq_log_info
38 #define vchiq_log_info(cat, fmt, ...) \
39 do { if (cat >= VCHIQ_LOG_INFO) \
40 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
42 #ifndef vchiq_log_trace
43 #define vchiq_log_trace(cat, fmt, ...) \
44 do { if (cat >= VCHIQ_LOG_TRACE) \
45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
48 #define vchiq_loud_error(...) \
49 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
51 #ifndef vchiq_static_assert
52 #define vchiq_static_assert(cond) __attribute__((unused)) \
53 extern int vchiq_static_assert[(cond) ? 1 : -1]
56 #define IS_POW2(x) (x && ((x & (x - 1)) == 0))
58 /* Ensure that the slot size and maximum number of slots are powers of 2 */
59 vchiq_static_assert(IS_POW2(VCHIQ_SLOT_SIZE));
60 vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS));
61 vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS_PER_SIDE));
63 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1)
64 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
65 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \
68 #define VCHIQ_MSG_PADDING 0 /* - */
69 #define VCHIQ_MSG_CONNECT 1 /* - */
70 #define VCHIQ_MSG_OPEN 2 /* + (srcport, -), fourcc, client_id */
71 #define VCHIQ_MSG_OPENACK 3 /* + (srcport, dstport) */
72 #define VCHIQ_MSG_CLOSE 4 /* + (srcport, dstport) */
73 #define VCHIQ_MSG_DATA 5 /* + (srcport, dstport) */
74 #define VCHIQ_MSG_BULK_RX 6 /* + (srcport, dstport), data, size */
75 #define VCHIQ_MSG_BULK_TX 7 /* + (srcport, dstport), data, size */
76 #define VCHIQ_MSG_BULK_RX_DONE 8 /* + (srcport, dstport), actual */
77 #define VCHIQ_MSG_BULK_TX_DONE 9 /* + (srcport, dstport), actual */
78 #define VCHIQ_MSG_PAUSE 10 /* - */
79 #define VCHIQ_MSG_RESUME 11 /* - */
80 #define VCHIQ_MSG_REMOTE_USE 12 /* - */
81 #define VCHIQ_MSG_REMOTE_RELEASE 13 /* - */
82 #define VCHIQ_MSG_REMOTE_USE_ACTIVE 14 /* - */
84 #define VCHIQ_PORT_MAX (VCHIQ_MAX_SERVICES - 1)
85 #define VCHIQ_PORT_FREE 0x1000
86 #define VCHIQ_PORT_IS_VALID(port) (port < VCHIQ_PORT_FREE)
87 #define VCHIQ_MAKE_MSG(type, srcport, dstport) \
88 ((type<<24) | (srcport<<12) | (dstport<<0))
89 #define VCHIQ_MSG_TYPE(msgid) ((unsigned int)msgid >> 24)
90 #define VCHIQ_MSG_SRCPORT(msgid) \
91 (unsigned short)(((unsigned int)msgid >> 12) & 0xfff)
92 #define VCHIQ_MSG_DSTPORT(msgid) \
93 ((unsigned short)msgid & 0xfff)
95 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \
96 ((fourcc) >> 24) & 0xff, \
97 ((fourcc) >> 16) & 0xff, \
98 ((fourcc) >> 8) & 0xff, \
101 /* Ensure the fields are wide enough */
102 vchiq_static_assert(VCHIQ_MSG_SRCPORT(VCHIQ_MAKE_MSG(0, 0, VCHIQ_PORT_MAX))
104 vchiq_static_assert(VCHIQ_MSG_TYPE(VCHIQ_MAKE_MSG(0, VCHIQ_PORT_MAX, 0)) == 0);
105 vchiq_static_assert((unsigned int)VCHIQ_PORT_MAX <
106 (unsigned int)VCHIQ_PORT_FREE);
108 #define VCHIQ_MSGID_PADDING VCHIQ_MAKE_MSG(VCHIQ_MSG_PADDING, 0, 0)
109 #define VCHIQ_MSGID_CLAIMED 0x40000000
111 #define VCHIQ_FOURCC_INVALID 0x00000000
112 #define VCHIQ_FOURCC_IS_LEGAL(fourcc) (fourcc != VCHIQ_FOURCC_INVALID)
114 #define VCHIQ_BULK_ACTUAL_ABORTED -1
116 typedef uint32_t BITSET_T;
118 vchiq_static_assert((sizeof(BITSET_T) * 8) == 32);
120 #define BITSET_SIZE(b) ((b + 31) >> 5)
121 #define BITSET_WORD(b) (b >> 5)
122 #define BITSET_BIT(b) (1 << (b & 31))
123 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b))
124 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
125 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
127 #if VCHIQ_ENABLE_STATS
128 #define VCHIQ_STATS_INC(state, stat) (state->stats. stat++)
129 #define VCHIQ_SERVICE_STATS_INC(service, stat) (service->stats. stat++)
130 #define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) \
131 (service->stats. stat += addend)
133 #define VCHIQ_STATS_INC(state, stat) ((void)0)
134 #define VCHIQ_SERVICE_STATS_INC(service, stat) ((void)0)
135 #define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) ((void)0)
140 #if VCHIQ_ENABLE_DEBUG
141 DEBUG_SLOT_HANDLER_COUNT,
142 DEBUG_SLOT_HANDLER_LINE,
146 DEBUG_AWAIT_COMPLETION_LINE,
147 DEBUG_DEQUEUE_MESSAGE_LINE,
148 DEBUG_SERVICE_CALLBACK_LINE,
149 DEBUG_MSG_QUEUE_FULL_COUNT,
150 DEBUG_COMPLETION_QUEUE_FULL_COUNT,
155 #if VCHIQ_ENABLE_DEBUG
157 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
158 #define DEBUG_TRACE(d) \
159 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
160 #define DEBUG_VALUE(d, v) \
161 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
162 #define DEBUG_COUNT(d) \
163 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
165 #else /* VCHIQ_ENABLE_DEBUG */
167 #define DEBUG_INITIALISE(local)
168 #define DEBUG_TRACE(d)
169 #define DEBUG_VALUE(d, v)
170 #define DEBUG_COUNT(d)
172 #endif /* VCHIQ_ENABLE_DEBUG */
174 enum vchiq_connstate {
175 VCHIQ_CONNSTATE_DISCONNECTED,
176 VCHIQ_CONNSTATE_CONNECTING,
177 VCHIQ_CONNSTATE_CONNECTED,
178 VCHIQ_CONNSTATE_PAUSING,
179 VCHIQ_CONNSTATE_PAUSE_SENT,
180 VCHIQ_CONNSTATE_PAUSED,
181 VCHIQ_CONNSTATE_RESUMING,
182 VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
183 VCHIQ_CONNSTATE_RESUME_TIMEOUT
188 VCHIQ_SRVSTATE_HIDDEN,
189 VCHIQ_SRVSTATE_LISTENING,
190 VCHIQ_SRVSTATE_OPENING,
192 VCHIQ_SRVSTATE_OPENSYNC,
193 VCHIQ_SRVSTATE_CLOSESENT,
194 VCHIQ_SRVSTATE_CLOSERECVD,
195 VCHIQ_SRVSTATE_CLOSEWAIT,
196 VCHIQ_SRVSTATE_CLOSED
200 VCHIQ_POLL_TERMINATE,
207 enum vchiq_bulk_dir {
212 typedef void (*vchiq_userdata_term)(void *userdata);
225 struct vchiq_bulk_queue {
226 int local_insert; /* Where to insert the next local bulk */
227 int remote_insert; /* Where to insert the next remote bulk (master) */
228 int process; /* Bulk to transfer next */
229 int remote_notify; /* Bulk to notify the remote client of next (mstr) */
230 int remove; /* Bulk to notify the local client of, and remove,
232 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
235 struct remote_event {
241 struct opaque_platform_state;
244 char data[VCHIQ_SLOT_SIZE];
247 struct vchiq_slot_info {
248 /* Use two counters rather than one to avoid the need for a mutex. */
253 struct vchiq_service {
254 struct vchiq_service_base base;
256 struct kref ref_count;
259 vchiq_userdata_term userdata_term;
260 unsigned int localport;
261 unsigned int remoteport;
273 struct vchiq_state *state;
274 struct vchiq_instance *instance;
276 int service_use_count;
278 struct vchiq_bulk_queue bulk_tx;
279 struct vchiq_bulk_queue bulk_rx;
281 struct completion remove_event;
282 struct completion bulk_remove_event;
283 struct mutex bulk_mutex;
285 struct service_stats_struct {
294 int bulk_aborted_count;
295 uint64_t ctrl_tx_bytes;
296 uint64_t ctrl_rx_bytes;
297 uint64_t bulk_tx_bytes;
298 uint64_t bulk_rx_bytes;
302 /* The quota information is outside struct vchiq_service so that it can
303 * be statically allocated, since for accounting reasons a service's slot
304 * usage is carried over between users of the same port number.
306 struct vchiq_service_quota {
307 unsigned short slot_quota;
308 unsigned short slot_use_count;
309 unsigned short message_quota;
310 unsigned short message_use_count;
311 struct completion quota_event;
312 int previous_tx_index;
315 struct vchiq_shared_state {
317 /* A non-zero value here indicates that the content is valid. */
320 /* The first and last (inclusive) slots allocated to the owner. */
324 /* The slot allocated to synchronous messages from the owner. */
327 /* Signalling this event indicates that owner's slot handler thread
329 struct remote_event trigger;
331 /* Indicates the byte position within the stream where the next message
332 ** will be written. The least significant bits are an index into the
333 ** slot. The next bits are the index of the slot in slot_queue. */
336 /* This event should be signalled when a slot is recycled. */
337 struct remote_event recycle;
339 /* The slot_queue index where the next recycled slot will be written. */
340 int slot_queue_recycle;
342 /* This event should be signalled when a synchronous message is sent. */
343 struct remote_event sync_trigger;
345 /* This event should be signalled when a synchronous message has been
347 struct remote_event sync_release;
349 /* A circular buffer of slot indexes. */
350 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
352 /* Debugging state */
353 int debug[DEBUG_MAX];
356 struct vchiq_slot_zero {
363 int max_slots_per_side;
364 int platform_data[2];
365 struct vchiq_shared_state master;
366 struct vchiq_shared_state slave;
367 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
373 enum vchiq_connstate conn_state;
374 short version_common;
376 struct vchiq_shared_state *local;
377 struct vchiq_shared_state *remote;
378 struct vchiq_slot *slot_data;
380 unsigned short default_slot_quota;
381 unsigned short default_message_quota;
383 /* Event indicating connect message received */
384 struct completion connect;
386 /* Mutex protecting services */
388 struct vchiq_instance **instance;
390 /* Processes incoming messages */
391 struct task_struct *slot_handler_thread;
393 /* Processes recycled slots */
394 struct task_struct *recycle_thread;
396 /* Processes synchronous messages */
397 struct task_struct *sync_thread;
399 /* Local implementation of the trigger remote event */
400 wait_queue_head_t trigger_event;
402 /* Local implementation of the recycle remote event */
403 wait_queue_head_t recycle_event;
405 /* Local implementation of the sync trigger remote event */
406 wait_queue_head_t sync_trigger_event;
408 /* Local implementation of the sync release remote event */
409 wait_queue_head_t sync_release_event;
413 struct vchiq_slot_info *rx_info;
415 struct mutex slot_mutex;
417 struct mutex recycle_mutex;
419 struct mutex sync_mutex;
421 struct mutex bulk_transfer_mutex;
423 /* Indicates the byte position within the stream from where the next
424 ** message will be read. The least significant bits are an index into
425 ** the slot.The next bits are the index of the slot in
426 ** remote->slot_queue. */
429 /* A cached copy of local->tx_pos. Only write to local->tx_pos, and read
430 from remote->tx_pos. */
433 /* The slot_queue index of the slot to become available next. */
434 int slot_queue_available;
436 /* A flag to indicate if any poll has been requested */
439 /* Ths index of the previous slot used for data messages. */
440 int previous_data_index;
442 /* The number of slots occupied by data messages. */
443 unsigned short data_use_count;
445 /* The maximum number of slots to be occupied by data messages. */
446 unsigned short data_quota;
448 /* An array of bit sets indicating which services must be polled. */
449 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
451 /* The number of the first unused service */
454 /* Signalled when a free slot becomes available. */
455 struct completion slot_available_event;
457 struct completion slot_remove_event;
459 /* Signalled when a free data slot becomes available. */
460 struct completion data_quota_event;
462 struct state_stats_struct {
470 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES];
471 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
472 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
474 struct opaque_platform_state *platform_state;
478 struct vchiq_bulk *bulk;
479 struct completion event;
483 extern spinlock_t bulk_waiter_spinlock;
485 extern int vchiq_core_log_level;
486 extern int vchiq_core_msg_log_level;
487 extern int vchiq_sync_log_level;
489 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES];
492 get_conn_state_name(enum vchiq_connstate conn_state);
494 extern struct vchiq_slot_zero *
495 vchiq_init_slots(void *mem_base, int mem_size);
497 extern enum vchiq_status
498 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero);
500 extern enum vchiq_status
501 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance);
503 extern struct vchiq_service *
504 vchiq_add_service_internal(struct vchiq_state *state,
505 const struct vchiq_service_params *params,
506 int srvstate, struct vchiq_instance *instance,
507 vchiq_userdata_term userdata_term);
509 extern enum vchiq_status
510 vchiq_open_service_internal(struct vchiq_service *service, int client_id);
512 extern enum vchiq_status
513 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
516 vchiq_terminate_service_internal(struct vchiq_service *service);
519 vchiq_free_service_internal(struct vchiq_service *service);
521 extern enum vchiq_status
522 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance);
525 remote_event_pollall(struct vchiq_state *state);
527 extern enum vchiq_status
528 vchiq_bulk_transfer(unsigned int handle, void *offset, int size,
529 void *userdata, enum vchiq_bulk_mode mode,
530 enum vchiq_bulk_dir dir);
533 vchiq_dump_state(void *dump_context, struct vchiq_state *state);
536 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service);
539 vchiq_loud_error_header(void);
542 vchiq_loud_error_footer(void);
545 request_poll(struct vchiq_state *state, struct vchiq_service *service,
548 static inline struct vchiq_service *
549 handle_to_service(unsigned int handle)
551 int idx = handle & (VCHIQ_MAX_SERVICES - 1);
552 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
553 (VCHIQ_MAX_STATES - 1)];
557 return rcu_dereference(state->services[idx]);
560 extern struct vchiq_service *
561 find_service_by_handle(unsigned int handle);
563 extern struct vchiq_service *
564 find_service_by_port(struct vchiq_state *state, int localport);
566 extern struct vchiq_service *
567 find_service_for_instance(struct vchiq_instance *instance,
568 unsigned int handle);
570 extern struct vchiq_service *
571 find_closed_service_for_instance(struct vchiq_instance *instance,
572 unsigned int handle);
574 extern struct vchiq_service *
575 __next_service_by_instance(struct vchiq_state *state,
576 struct vchiq_instance *instance,
579 extern struct vchiq_service *
580 next_service_by_instance(struct vchiq_state *state,
581 struct vchiq_instance *instance,
585 lock_service(struct vchiq_service *service);
588 unlock_service(struct vchiq_service *service);
590 /* The following functions are called from vchiq_core, and external
591 ** implementations must be provided. */
593 extern enum vchiq_status
594 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
598 vchiq_complete_bulk(struct vchiq_bulk *bulk);
601 remote_event_signal(struct remote_event *event);
604 vchiq_dump(void *dump_context, const char *str, int len);
607 vchiq_dump_platform_state(void *dump_context);
610 vchiq_dump_platform_instances(void *dump_context);
613 vchiq_dump_platform_service_state(void *dump_context,
614 struct vchiq_service *service);
616 extern enum vchiq_status
617 vchiq_use_service_internal(struct vchiq_service *service);
619 extern enum vchiq_status
620 vchiq_release_service_internal(struct vchiq_service *service);
623 vchiq_on_remote_use(struct vchiq_state *state);
626 vchiq_on_remote_release(struct vchiq_state *state);
628 extern enum vchiq_status
629 vchiq_platform_init_state(struct vchiq_state *state);
631 extern enum vchiq_status
632 vchiq_check_service(struct vchiq_service *service);
635 vchiq_on_remote_use_active(struct vchiq_state *state);
637 extern enum vchiq_status
638 vchiq_send_remote_use(struct vchiq_state *state);
640 extern enum vchiq_status
641 vchiq_send_remote_use_active(struct vchiq_state *state);
644 vchiq_platform_conn_state_changed(struct vchiq_state *state,
645 enum vchiq_connstate oldstate,
646 enum vchiq_connstate newstate);
649 vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate);
652 vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,