Zephyr Project API 3.7.0
A Scalable Open Source RTOS
Loading...
Searching...
No Matches
rtio.h
Go to the documentation of this file.
1/*
2 * Copyright (c) 2022 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
26#ifndef ZEPHYR_INCLUDE_RTIO_RTIO_H_
27#define ZEPHYR_INCLUDE_RTIO_RTIO_H_
28
29#include <string.h>
30
32#include <zephyr/device.h>
33#include <zephyr/kernel.h>
34#include <zephyr/sys/__assert.h>
35#include <zephyr/sys/atomic.h>
37#include <zephyr/sys/util.h>
40
41#ifdef __cplusplus
42extern "C" {
43#endif
44
45
65#define RTIO_PRIO_LOW 0U
66
70#define RTIO_PRIO_NORM 127U
71
75#define RTIO_PRIO_HIGH 255U
76
96#define RTIO_SQE_CHAINED BIT(0)
97
108#define RTIO_SQE_TRANSACTION BIT(1)
109
110
120#define RTIO_SQE_MEMPOOL_BUFFER BIT(2)
121
128#define RTIO_SQE_CANCELED BIT(3)
129
136#define RTIO_SQE_MULTISHOT BIT(4)
137
141#define RTIO_SQE_NO_RESPONSE BIT(5)
142
160#define RTIO_CQE_FLAG_MEMPOOL_BUFFER BIT(0)
161
162#define RTIO_CQE_FLAG_GET(flags) FIELD_GET(GENMASK(7, 0), (flags))
163
170#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(flags) FIELD_GET(GENMASK(19, 8), (flags))
171
178#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(flags) FIELD_GET(GENMASK(31, 20), (flags))
179
187#define RTIO_CQE_FLAG_PREP_MEMPOOL(blk_idx, blk_cnt) \
188 (FIELD_PREP(GENMASK(7, 0), RTIO_CQE_FLAG_MEMPOOL_BUFFER) | \
189 FIELD_PREP(GENMASK(19, 8), blk_idx) | FIELD_PREP(GENMASK(31, 20), blk_cnt))
190
198#define RTIO_IODEV_I2C_STOP BIT(1)
199
203#define RTIO_IODEV_I2C_RESTART BIT(2)
204
208#define RTIO_IODEV_I2C_10_BITS BIT(3)
209
211struct rtio;
212struct rtio_cqe;
213struct rtio_sqe;
214struct rtio_sqe_pool;
215struct rtio_cqe_pool;
216struct rtio_iodev;
217struct rtio_iodev_sqe;
227typedef void (*rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, void *arg0);
228
232struct rtio_sqe {
241 uint16_t _resv0;
242
243 const struct rtio_iodev *iodev;
252 void *userdata;
253
254 union {
255
257 struct {
260 };
261
263 struct {
265 uint8_t tiny_buf[7];
266 };
267
269 struct {
271 void *arg0;
272 };
273
275 struct {
279 };
280
283 };
284};
285
287/* Ensure the rtio_sqe never grows beyond a common cacheline size of 64 bytes */
288BUILD_ASSERT(sizeof(struct rtio_sqe) <= 64);
301
308
315
327struct rtio {
328#ifdef CONFIG_RTIO_SUBMIT_SEM
329 /* A wait semaphore which may suspend the calling thread
330 * to wait for some number of completions when calling submit
331 */
332 struct k_sem *submit_sem;
333
334 uint32_t submit_count;
335#endif
336
337#ifdef CONFIG_RTIO_CONSUME_SEM
338 /* A wait semaphore which may suspend the calling thread
339 * to wait for some number of completions while consuming
340 * them from the completion queue
341 */
342 struct k_sem *consume_sem;
343#endif
344
345 /* Total number of completions */
347
348 /* Number of completions that were unable to be submitted with results
349 * due to the cq spsc being full
350 */
352
353 /* Submission queue object pool with free list */
355
356 /* Complete queue object pool with free list */
358
359#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
360 /* Mem block pool */
361 struct sys_mem_blocks *block_pool;
362#endif
363
364 /* Submission queue */
365 struct mpsc sq;
366
367 /* Completion queue */
368 struct mpsc cq;
369};
370
372extern struct k_mem_partition rtio_partition;
373
381static inline size_t rtio_mempool_block_size(const struct rtio *r)
382{
383#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
384 ARG_UNUSED(r);
385 return 0;
386#else
387 if (r == NULL || r->block_pool == NULL) {
388 return 0;
389 }
390 return BIT(r->block_pool->info.blk_sz_shift);
391#endif
392}
393
401#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
402static inline uint16_t __rtio_compute_mempool_block_index(const struct rtio *r, const void *ptr)
403{
404 uintptr_t addr = (uintptr_t)ptr;
405 struct sys_mem_blocks *mem_pool = r->block_pool;
406 uint32_t block_size = rtio_mempool_block_size(r);
407
408 uintptr_t buff = (uintptr_t)mem_pool->buffer;
409 uint32_t buff_size = mem_pool->info.num_blocks * block_size;
410
411 if (addr < buff || addr >= buff + buff_size) {
412 return UINT16_MAX;
413 }
414 return (addr - buff) / block_size;
415}
416#endif
417
424 struct rtio_sqe sqe;
425 struct mpsc_node q;
427 struct rtio *r;
428};
429
442 void (*submit)(struct rtio_iodev_sqe *iodev_sqe);
443};
444
449 /* Function pointer table */
450 const struct rtio_iodev_api *api;
451
452 /* Data associated with this iodev */
453 void *data;
454};
455
457#define RTIO_OP_NOP 0
458
460#define RTIO_OP_RX (RTIO_OP_NOP+1)
461
463#define RTIO_OP_TX (RTIO_OP_RX+1)
464
466#define RTIO_OP_TINY_TX (RTIO_OP_TX+1)
467
469#define RTIO_OP_CALLBACK (RTIO_OP_TINY_TX+1)
470
472#define RTIO_OP_TXRX (RTIO_OP_CALLBACK+1)
473
475#define RTIO_OP_I2C_RECOVER (RTIO_OP_TXRX+1)
476
478#define RTIO_OP_I2C_CONFIGURE (RTIO_OP_I2C_RECOVER+1)
479
483static inline void rtio_sqe_prep_nop(struct rtio_sqe *sqe,
484 const struct rtio_iodev *iodev,
485 void *userdata)
486{
487 memset(sqe, 0, sizeof(struct rtio_sqe));
488 sqe->op = RTIO_OP_NOP;
489 sqe->iodev = iodev;
490 sqe->userdata = userdata;
491}
492
496static inline void rtio_sqe_prep_read(struct rtio_sqe *sqe,
497 const struct rtio_iodev *iodev,
498 int8_t prio,
499 uint8_t *buf,
500 uint32_t len,
501 void *userdata)
502{
503 memset(sqe, 0, sizeof(struct rtio_sqe));
504 sqe->op = RTIO_OP_RX;
505 sqe->prio = prio;
506 sqe->iodev = iodev;
507 sqe->buf_len = len;
508 sqe->buf = buf;
509 sqe->userdata = userdata;
510}
511
517static inline void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe,
518 const struct rtio_iodev *iodev, int8_t prio,
519 void *userdata)
520{
521 rtio_sqe_prep_read(sqe, iodev, prio, NULL, 0, userdata);
523}
524
525static inline void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe,
526 const struct rtio_iodev *iodev, int8_t prio,
527 void *userdata)
528{
529 rtio_sqe_prep_read_with_pool(sqe, iodev, prio, userdata);
531}
532
536static inline void rtio_sqe_prep_write(struct rtio_sqe *sqe,
537 const struct rtio_iodev *iodev,
538 int8_t prio,
539 uint8_t *buf,
540 uint32_t len,
541 void *userdata)
542{
543 memset(sqe, 0, sizeof(struct rtio_sqe));
544 sqe->op = RTIO_OP_TX;
545 sqe->prio = prio;
546 sqe->iodev = iodev;
547 sqe->buf_len = len;
548 sqe->buf = buf;
549 sqe->userdata = userdata;
550}
551
562static inline void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe,
563 const struct rtio_iodev *iodev,
564 int8_t prio,
565 const uint8_t *tiny_write_data,
566 uint8_t tiny_write_len,
567 void *userdata)
568{
569 __ASSERT_NO_MSG(tiny_write_len <= sizeof(sqe->tiny_buf));
570
571 memset(sqe, 0, sizeof(struct rtio_sqe));
572 sqe->op = RTIO_OP_TINY_TX;
573 sqe->prio = prio;
574 sqe->iodev = iodev;
575 sqe->tiny_buf_len = tiny_write_len;
576 memcpy(sqe->tiny_buf, tiny_write_data, tiny_write_len);
577 sqe->userdata = userdata;
578}
579
588static inline void rtio_sqe_prep_callback(struct rtio_sqe *sqe,
589 rtio_callback_t callback,
590 void *arg0,
591 void *userdata)
592{
593 memset(sqe, 0, sizeof(struct rtio_sqe));
594 sqe->op = RTIO_OP_CALLBACK;
595 sqe->prio = 0;
596 sqe->iodev = NULL;
597 sqe->callback = callback;
598 sqe->arg0 = arg0;
599 sqe->userdata = userdata;
600}
601
605static inline void rtio_sqe_prep_transceive(struct rtio_sqe *sqe,
606 const struct rtio_iodev *iodev,
607 int8_t prio,
608 uint8_t *tx_buf,
609 uint8_t *rx_buf,
610 uint32_t buf_len,
611 void *userdata)
612{
613 memset(sqe, 0, sizeof(struct rtio_sqe));
614 sqe->op = RTIO_OP_TXRX;
615 sqe->prio = prio;
616 sqe->iodev = iodev;
617 sqe->txrx_buf_len = buf_len;
618 sqe->tx_buf = tx_buf;
619 sqe->rx_buf = rx_buf;
620 sqe->userdata = userdata;
621}
622
623static inline struct rtio_iodev_sqe *rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
624{
625 struct mpsc_node *node = mpsc_pop(&pool->free_q);
626
627 if (node == NULL) {
628 return NULL;
629 }
630
631 struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
632
633 pool->pool_free--;
634
635 return iodev_sqe;
636}
637
638static inline void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
639{
640 mpsc_push(&pool->free_q, &iodev_sqe->q);
641
642 pool->pool_free++;
643}
644
645static inline struct rtio_cqe *rtio_cqe_pool_alloc(struct rtio_cqe_pool *pool)
646{
647 struct mpsc_node *node = mpsc_pop(&pool->free_q);
648
649 if (node == NULL) {
650 return NULL;
651 }
652
653 struct rtio_cqe *cqe = CONTAINER_OF(node, struct rtio_cqe, q);
654
655 memset(cqe, 0, sizeof(struct rtio_cqe));
656
657 pool->pool_free--;
658
659 return cqe;
660}
661
662static inline void rtio_cqe_pool_free(struct rtio_cqe_pool *pool, struct rtio_cqe *cqe)
663{
664 mpsc_push(&pool->free_q, &cqe->q);
665
666 pool->pool_free++;
667}
668
669static inline int rtio_block_pool_alloc(struct rtio *r, size_t min_sz,
670 size_t max_sz, uint8_t **buf, uint32_t *buf_len)
671{
672#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
673 ARG_UNUSED(r);
674 ARG_UNUSED(min_sz);
675 ARG_UNUSED(max_sz);
676 ARG_UNUSED(buf);
677 ARG_UNUSED(buf_len);
678 return -ENOTSUP;
679#else
680 const uint32_t block_size = rtio_mempool_block_size(r);
681 uint32_t bytes = max_sz;
682
683 /* Not every context has a block pool and the block size may return 0 in
684 * that case
685 */
686 if (block_size == 0) {
687 return -ENOMEM;
688 }
689
690 do {
691 size_t num_blks = DIV_ROUND_UP(bytes, block_size);
692 int rc = sys_mem_blocks_alloc_contiguous(r->block_pool, num_blks, (void **)buf);
693
694 if (rc == 0) {
695 *buf_len = num_blks * block_size;
696 return 0;
697 }
698
699 bytes -= block_size;
700 } while (bytes >= min_sz);
701
702 return -ENOMEM;
703#endif
704}
705
706static inline void rtio_block_pool_free(struct rtio *r, void *buf, uint32_t buf_len)
707{
708#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
709 ARG_UNUSED(r);
710 ARG_UNUSED(buf);
711 ARG_UNUSED(buf_len);
712#else
713 size_t num_blks = buf_len >> r->block_pool->info.blk_sz_shift;
714
715 sys_mem_blocks_free_contiguous(r->block_pool, buf, num_blks);
716#endif
717}
718
719/* Do not try and reformat the macros */
720/* clang-format off */
721
729#define RTIO_IODEV_DEFINE(name, iodev_api, iodev_data) \
730 STRUCT_SECTION_ITERABLE(rtio_iodev, name) = { \
731 .api = (iodev_api), \
732 .data = (iodev_data), \
733 }
734
735#define Z_RTIO_SQE_POOL_DEFINE(name, sz) \
736 static struct rtio_iodev_sqe CONCAT(_sqe_pool_, name)[sz]; \
737 STRUCT_SECTION_ITERABLE(rtio_sqe_pool, name) = { \
738 .free_q = MPSC_INIT((name.free_q)), \
739 .pool_size = sz, \
740 .pool_free = sz, \
741 .pool = CONCAT(_sqe_pool_, name), \
742 }
743
744
745#define Z_RTIO_CQE_POOL_DEFINE(name, sz) \
746 static struct rtio_cqe CONCAT(_cqe_pool_, name)[sz]; \
747 STRUCT_SECTION_ITERABLE(rtio_cqe_pool, name) = { \
748 .free_q = MPSC_INIT((name.free_q)), \
749 .pool_size = sz, \
750 .pool_free = sz, \
751 .pool = CONCAT(_cqe_pool_, name), \
752 }
753
763#define RTIO_BMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_BMEM(rtio_partition) static), (static))
764
774#define RTIO_DMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_DMEM(rtio_partition) static), (static))
775
776#define Z_RTIO_BLOCK_POOL_DEFINE(name, blk_sz, blk_cnt, blk_align) \
777 RTIO_BMEM uint8_t __aligned(WB_UP(blk_align)) \
778 CONCAT(_block_pool_, name)[blk_cnt*WB_UP(blk_sz)]; \
779 _SYS_MEM_BLOCKS_DEFINE_WITH_EXT_BUF(name, WB_UP(blk_sz), blk_cnt, \
780 CONCAT(_block_pool_, name), RTIO_DMEM)
781
782#define Z_RTIO_DEFINE(name, _sqe_pool, _cqe_pool, _block_pool) \
783 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, \
784 (static K_SEM_DEFINE(CONCAT(_submit_sem_, name), 0, K_SEM_MAX_LIMIT))) \
785 IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, \
786 (static K_SEM_DEFINE(CONCAT(_consume_sem_, name), 0, K_SEM_MAX_LIMIT))) \
787 STRUCT_SECTION_ITERABLE(rtio, name) = { \
788 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_sem = &CONCAT(_submit_sem_, name),)) \
789 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_count = 0,)) \
790 IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, (.consume_sem = &CONCAT(_consume_sem_, name),))\
791 .cq_count = ATOMIC_INIT(0), \
792 .xcqcnt = ATOMIC_INIT(0), \
793 .sqe_pool = _sqe_pool, \
794 .cqe_pool = _cqe_pool, \
795 IF_ENABLED(CONFIG_RTIO_SYS_MEM_BLOCKS, (.block_pool = _block_pool,)) \
796 .sq = MPSC_INIT((name.sq)), \
797 .cq = MPSC_INIT((name.cq)), \
798 }
799
807#define RTIO_DEFINE(name, sq_sz, cq_sz) \
808 Z_RTIO_SQE_POOL_DEFINE(CONCAT(name, _sqe_pool), sq_sz); \
809 Z_RTIO_CQE_POOL_DEFINE(CONCAT(name, _cqe_pool), cq_sz); \
810 Z_RTIO_DEFINE(name, &CONCAT(name, _sqe_pool), \
811 &CONCAT(name, _cqe_pool), NULL)
812
813/* clang-format on */
814
825#define RTIO_DEFINE_WITH_MEMPOOL(name, sq_sz, cq_sz, num_blks, blk_size, balign) \
826 Z_RTIO_SQE_POOL_DEFINE(name##_sqe_pool, sq_sz); \
827 Z_RTIO_CQE_POOL_DEFINE(name##_cqe_pool, cq_sz); \
828 Z_RTIO_BLOCK_POOL_DEFINE(name##_block_pool, blk_size, num_blks, balign); \
829 Z_RTIO_DEFINE(name, &name##_sqe_pool, &name##_cqe_pool, &name##_block_pool)
830
831/* clang-format on */
832
840static inline uint32_t rtio_sqe_acquirable(struct rtio *r)
841{
842 return r->sqe_pool->pool_free;
843}
844
853static inline struct rtio_iodev_sqe *rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
854{
855 if (iodev_sqe->sqe.flags & RTIO_SQE_TRANSACTION) {
856 return iodev_sqe->next;
857 } else {
858 return NULL;
859 }
860}
861
862
871static inline struct rtio_iodev_sqe *rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
872{
873 if (iodev_sqe->sqe.flags & RTIO_SQE_CHAINED) {
874 return iodev_sqe->next;
875 } else {
876 return NULL;
877 }
878}
879
888static inline struct rtio_iodev_sqe *rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
889{
890 return iodev_sqe->next;
891}
892
901static inline struct rtio_sqe *rtio_sqe_acquire(struct rtio *r)
902{
903 struct rtio_iodev_sqe *iodev_sqe = rtio_sqe_pool_alloc(r->sqe_pool);
904
905 if (iodev_sqe == NULL) {
906 return NULL;
907 }
908
909 mpsc_push(&r->sq, &iodev_sqe->q);
910
911 return &iodev_sqe->sqe;
912}
913
919static inline void rtio_sqe_drop_all(struct rtio *r)
920{
921 struct rtio_iodev_sqe *iodev_sqe;
922 struct mpsc_node *node = mpsc_pop(&r->sq);
923
924 while (node != NULL) {
925 iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
926 rtio_sqe_pool_free(r->sqe_pool, iodev_sqe);
927 node = mpsc_pop(&r->sq);
928 }
929}
930
934static inline struct rtio_cqe *rtio_cqe_acquire(struct rtio *r)
935{
936 struct rtio_cqe *cqe = rtio_cqe_pool_alloc(r->cqe_pool);
937
938 if (cqe == NULL) {
939 return NULL;
940 }
941
942 memset(cqe, 0, sizeof(struct rtio_cqe));
943
944 return cqe;
945}
946
950static inline void rtio_cqe_produce(struct rtio *r, struct rtio_cqe *cqe)
951{
952 mpsc_push(&r->cq, &cqe->q);
953}
954
966static inline struct rtio_cqe *rtio_cqe_consume(struct rtio *r)
967{
968 struct mpsc_node *node;
969 struct rtio_cqe *cqe = NULL;
970
971#ifdef CONFIG_RTIO_CONSUME_SEM
972 if (k_sem_take(r->consume_sem, K_NO_WAIT) != 0) {
973 return NULL;
974 }
975#endif
976
977 node = mpsc_pop(&r->cq);
978 if (node == NULL) {
979 return NULL;
980 }
981 cqe = CONTAINER_OF(node, struct rtio_cqe, q);
982
983 return cqe;
984}
985
996static inline struct rtio_cqe *rtio_cqe_consume_block(struct rtio *r)
997{
998 struct mpsc_node *node;
999 struct rtio_cqe *cqe;
1000
1001#ifdef CONFIG_RTIO_CONSUME_SEM
1002 k_sem_take(r->consume_sem, K_FOREVER);
1003#endif
1004 node = mpsc_pop(&r->cq);
1005 while (node == NULL) {
1006 Z_SPIN_DELAY(1);
1007 node = mpsc_pop(&r->cq);
1008 }
1009 cqe = CONTAINER_OF(node, struct rtio_cqe, q);
1010
1011 return cqe;
1012}
1013
1020static inline void rtio_cqe_release(struct rtio *r, struct rtio_cqe *cqe)
1021{
1022 rtio_cqe_pool_free(r->cqe_pool, cqe);
1023}
1024
1031static inline uint32_t rtio_cqe_compute_flags(struct rtio_iodev_sqe *iodev_sqe)
1032{
1033 uint32_t flags = 0;
1034
1035#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1036 if (iodev_sqe->sqe.op == RTIO_OP_RX && iodev_sqe->sqe.flags & RTIO_SQE_MEMPOOL_BUFFER) {
1037 struct rtio *r = iodev_sqe->r;
1038 struct sys_mem_blocks *mem_pool = r->block_pool;
1039 int blk_index = (iodev_sqe->sqe.buf - mem_pool->buffer) >>
1040 mem_pool->info.blk_sz_shift;
1041 int blk_count = iodev_sqe->sqe.buf_len >> mem_pool->info.blk_sz_shift;
1042
1043 flags = RTIO_CQE_FLAG_PREP_MEMPOOL(blk_index, blk_count);
1044 }
1045#else
1046 ARG_UNUSED(iodev_sqe);
1047#endif
1048
1049 return flags;
1050}
1051
1067__syscall int rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1068 uint8_t **buff, uint32_t *buff_len);
1069
1070static inline int z_impl_rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1071 uint8_t **buff, uint32_t *buff_len)
1072{
1073#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1075 int blk_idx = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(cqe->flags);
1076 int blk_count = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(cqe->flags);
1078
1079 *buff = r->block_pool->buffer + blk_idx * blk_size;
1080 *buff_len = blk_count * blk_size;
1081 __ASSERT_NO_MSG(*buff >= r->block_pool->buffer);
1082 __ASSERT_NO_MSG(*buff <
1083 r->block_pool->buffer + blk_size * r->block_pool->info.num_blocks);
1084 return 0;
1085 }
1086 return -EINVAL;
1087#else
1088 ARG_UNUSED(r);
1089 ARG_UNUSED(cqe);
1090 ARG_UNUSED(buff);
1091 ARG_UNUSED(buff_len);
1092
1093 return -ENOTSUP;
1094#endif
1095}
1096
1098void rtio_executor_ok(struct rtio_iodev_sqe *iodev_sqe, int result);
1099void rtio_executor_err(struct rtio_iodev_sqe *iodev_sqe, int result);
1100
1109static inline void rtio_iodev_sqe_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
1110{
1111 rtio_executor_ok(iodev_sqe, result);
1112}
1113
1122static inline void rtio_iodev_sqe_err(struct rtio_iodev_sqe *iodev_sqe, int result)
1123{
1124 rtio_executor_err(iodev_sqe, result);
1125}
1126
1138static inline void rtio_cqe_submit(struct rtio *r, int result, void *userdata, uint32_t flags)
1139{
1140 struct rtio_cqe *cqe = rtio_cqe_acquire(r);
1141
1142 if (cqe == NULL) {
1143 atomic_inc(&r->xcqcnt);
1144 } else {
1145 cqe->result = result;
1146 cqe->userdata = userdata;
1147 cqe->flags = flags;
1148 rtio_cqe_produce(r, cqe);
1149 }
1150
1151 atomic_inc(&r->cq_count);
1152#ifdef CONFIG_RTIO_SUBMIT_SEM
1153 if (r->submit_count > 0) {
1154 r->submit_count--;
1155 if (r->submit_count == 0) {
1156 k_sem_give(r->submit_sem);
1157 }
1158 }
1159#endif
1160#ifdef CONFIG_RTIO_CONSUME_SEM
1161 k_sem_give(r->consume_sem);
1162#endif
1163}
1164
1165#define __RTIO_MEMPOOL_GET_NUM_BLKS(num_bytes, blk_size) (((num_bytes) + (blk_size)-1) / (blk_size))
1166
1179static inline int rtio_sqe_rx_buf(const struct rtio_iodev_sqe *iodev_sqe, uint32_t min_buf_len,
1180 uint32_t max_buf_len, uint8_t **buf, uint32_t *buf_len)
1181{
1182 struct rtio_sqe *sqe = (struct rtio_sqe *)&iodev_sqe->sqe;
1183
1184#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1185 if (sqe->op == RTIO_OP_RX && sqe->flags & RTIO_SQE_MEMPOOL_BUFFER) {
1186 struct rtio *r = iodev_sqe->r;
1187
1188 if (sqe->buf != NULL) {
1189 if (sqe->buf_len < min_buf_len) {
1190 return -ENOMEM;
1191 }
1192 *buf = sqe->buf;
1193 *buf_len = sqe->buf_len;
1194 return 0;
1195 }
1196
1197 int rc = rtio_block_pool_alloc(r, min_buf_len, max_buf_len, buf, buf_len);
1198 if (rc == 0) {
1199 sqe->buf = *buf;
1200 sqe->buf_len = *buf_len;
1201 return 0;
1202 }
1203
1204 return -ENOMEM;
1205 }
1206#else
1207 ARG_UNUSED(max_buf_len);
1208#endif
1209
1210 if (sqe->buf_len < min_buf_len) {
1211 return -ENOMEM;
1212 }
1213
1214 *buf = sqe->buf;
1215 *buf_len = sqe->buf_len;
1216 return 0;
1217}
1218
1233__syscall void rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len);
1234
1235static inline void z_impl_rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len)
1236{
1237#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1238 if (r == NULL || buff == NULL || r->block_pool == NULL || buff_len == 0) {
1239 return;
1240 }
1241
1242 rtio_block_pool_free(r, buff, buff_len);
1243#else
1244 ARG_UNUSED(r);
1245 ARG_UNUSED(buff);
1246 ARG_UNUSED(buff_len);
1247#endif
1248}
1249
1253static inline void rtio_access_grant(struct rtio *r, struct k_thread *t)
1254{
1256
1257#ifdef CONFIG_RTIO_SUBMIT_SEM
1258 k_object_access_grant(r->submit_sem, t);
1259#endif
1260
1261#ifdef CONFIG_RTIO_CONSUME_SEM
1262 k_object_access_grant(r->consume_sem, t);
1263#endif
1264}
1265
1276__syscall int rtio_sqe_cancel(struct rtio_sqe *sqe);
1277
1278static inline int z_impl_rtio_sqe_cancel(struct rtio_sqe *sqe)
1279{
1280 struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe);
1281
1282 do {
1283 iodev_sqe->sqe.flags |= RTIO_SQE_CANCELED;
1284 iodev_sqe = rtio_iodev_sqe_next(iodev_sqe);
1285 } while (iodev_sqe != NULL);
1286
1287 return 0;
1288}
1289
1305__syscall int rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1306 struct rtio_sqe **handle, size_t sqe_count);
1307
1308static inline int z_impl_rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1309 struct rtio_sqe **handle,
1310 size_t sqe_count)
1311{
1312 struct rtio_sqe *sqe;
1313 uint32_t acquirable = rtio_sqe_acquirable(r);
1314
1315 if (acquirable < sqe_count) {
1316 return -ENOMEM;
1317 }
1318
1319 for (unsigned long i = 0; i < sqe_count; i++) {
1320 sqe = rtio_sqe_acquire(r);
1321 __ASSERT_NO_MSG(sqe != NULL);
1322 if (handle != NULL && i == 0) {
1323 *handle = sqe;
1324 }
1325 *sqe = sqes[i];
1326 }
1327
1328 return 0;
1329}
1330
1347static inline int rtio_sqe_copy_in(struct rtio *r, const struct rtio_sqe *sqes, size_t sqe_count)
1348{
1349 return rtio_sqe_copy_in_get_handles(r, sqes, NULL, sqe_count);
1350}
1351
1367__syscall int rtio_cqe_copy_out(struct rtio *r,
1368 struct rtio_cqe *cqes,
1369 size_t cqe_count,
1371static inline int z_impl_rtio_cqe_copy_out(struct rtio *r,
1372 struct rtio_cqe *cqes,
1373 size_t cqe_count,
1375{
1376 size_t copied = 0;
1377 struct rtio_cqe *cqe;
1379
1380 do {
1383 if (cqe == NULL) {
1384#ifdef CONFIG_BOARD_NATIVE_POSIX
1385 /* Native posix fakes the clock and only moves it forward when sleeping. */
1386 k_sleep(K_TICKS(1));
1387#else
1388 Z_SPIN_DELAY(1);
1389#endif
1390 continue;
1391 }
1392 cqes[copied++] = *cqe;
1393 rtio_cqe_release(r, cqe);
1394 } while (copied < cqe_count && !sys_timepoint_expired(end));
1395
1396 return copied;
1397}
1398
1412__syscall int rtio_submit(struct rtio *r, uint32_t wait_count);
1413
1414static inline int z_impl_rtio_submit(struct rtio *r, uint32_t wait_count)
1415{
1416 int res = 0;
1417
1418#ifdef CONFIG_RTIO_SUBMIT_SEM
1419 /* TODO undefined behavior if another thread calls submit of course
1420 */
1421 if (wait_count > 0) {
1422 __ASSERT(!k_is_in_isr(),
1423 "expected rtio submit with wait count to be called from a thread");
1424
1425 k_sem_reset(r->submit_sem);
1426 r->submit_count = wait_count;
1427 }
1428#else
1429 uintptr_t cq_count = (uintptr_t)atomic_get(&r->cq_count) + wait_count;
1430#endif
1431
1432 /* Submit the queue to the executor which consumes submissions
1433 * and produces completions through ISR chains or other means.
1434 */
1436
1437
1438 /* TODO could be nicer if we could suspend the thread and not
1439 * wake up on each completion here.
1440 */
1441#ifdef CONFIG_RTIO_SUBMIT_SEM
1442
1443 if (wait_count > 0) {
1444 res = k_sem_take(r->submit_sem, K_FOREVER);
1445 __ASSERT(res == 0,
1446 "semaphore was reset or timed out while waiting on completions!");
1447 }
1448#else
1449 while ((uintptr_t)atomic_get(&r->cq_count) < cq_count) {
1450 Z_SPIN_DELAY(10);
1451 k_yield();
1452 }
1453#endif
1454
1455 return res;
1456}
1457
1462#ifdef __cplusplus
1463}
1464#endif
1465
1466#include <zephyr/syscalls/rtio.h>
1467
1468#endif /* ZEPHYR_INCLUDE_RTIO_RTIO_H_ */
workaround assembler barfing for ST r
Definition asm-macro-32-bit-gnu.h:24
long atomic_t
Definition atomic_types.h:15
static ALWAYS_INLINE atomic_val_t atomic_inc(atomic_t *target)
Implementation of atomic_inc.
Definition atomic_xtensa.h:112
static ALWAYS_INLINE atomic_val_t atomic_get(const atomic_t *target)
Implementation of atomic_get.
Definition atomic_xtensa.h:18
ZTEST_BMEM int timeout
Definition main.c:31
#define K_FOREVER
Generate infinite timeout delay.
Definition kernel.h:1363
#define K_NO_WAIT
Generate null timeout delay.
Definition kernel.h:1253
k_timepoint_t sys_timepoint_calc(k_timeout_t timeout)
Calculate a timepoint value.
static bool sys_timepoint_expired(k_timepoint_t timepoint)
Indicates if timepoint is expired.
Definition sys_clock.h:327
#define K_TIMEOUT_EQ(a, b)
Compare timeouts for equality.
Definition sys_clock.h:80
#define K_TICKS(t)
Generate timeout delay from system ticks.
Definition kernel.h:1305
bool k_is_in_isr(void)
Determine if code is running at interrupt level.
int sys_mem_blocks_free_contiguous(sys_mem_blocks_t *mem_block, void *block, size_t count)
Free contiguous multiple memory blocks.
int sys_mem_blocks_alloc_contiguous(sys_mem_blocks_t *mem_block, size_t count, void **out_block)
Allocate a contiguous set of memory blocks.
static ALWAYS_INLINE void mpsc_push(struct mpsc *q, struct mpsc_node *n)
Push a node.
Definition mpsc_lockfree.h:126
static struct mpsc_node * mpsc_pop(struct mpsc *q)
Pop a node off of the list.
Definition mpsc_lockfree.h:145
#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(flags)
Get the block count of a mempool flags.
Definition rtio.h:178
#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(flags)
Get the block index of a mempool flags.
Definition rtio.h:170
#define RTIO_CQE_FLAG_MEMPOOL_BUFFER
The entry's buffer was allocated from the RTIO's mempool.
Definition rtio.h:160
#define RTIO_CQE_FLAG_PREP_MEMPOOL(blk_idx, blk_cnt)
Prepare CQE flags for a mempool read.
Definition rtio.h:187
#define RTIO_CQE_FLAG_GET(flags)
Definition rtio.h:162
#define RTIO_SQE_MULTISHOT
The SQE should continue producing CQEs until canceled.
Definition rtio.h:136
#define RTIO_SQE_TRANSACTION
The next request in the queue is part of a transaction.
Definition rtio.h:108
#define RTIO_SQE_MEMPOOL_BUFFER
The buffer should be allocated by the RTIO mempool.
Definition rtio.h:120
#define RTIO_SQE_CANCELED
The SQE should not execute if possible.
Definition rtio.h:128
#define RTIO_SQE_CHAINED
The next request in the queue should wait on this one.
Definition rtio.h:96
static void rtio_sqe_prep_transceive(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, uint8_t *tx_buf, uint8_t *rx_buf, uint32_t buf_len, void *userdata)
Prepare a transceive op submission.
Definition rtio.h:605
static void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, void *userdata)
Prepare a read op submission with context's mempool.
Definition rtio.h:517
void rtio_executor_err(struct rtio_iodev_sqe *iodev_sqe, int result)
#define RTIO_OP_CALLBACK
An operation that calls a given function (callback)
Definition rtio.h:469
static uint32_t rtio_sqe_acquirable(struct rtio *r)
Count of acquirable submission queue events.
Definition rtio.h:840
static void rtio_cqe_pool_free(struct rtio_cqe_pool *pool, struct rtio_cqe *cqe)
Definition rtio.h:662
static void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, const uint8_t *tiny_write_data, uint8_t tiny_write_len, void *userdata)
Prepare a tiny write op submission.
Definition rtio.h:562
static size_t rtio_mempool_block_size(const struct rtio *r)
Get the mempool block size of the RTIO context.
Definition rtio.h:381
static void rtio_cqe_submit(struct rtio *r, int result, void *userdata, uint32_t flags)
Submit a completion queue event with a given result and userdata.
Definition rtio.h:1138
static void rtio_sqe_prep_nop(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, void *userdata)
Prepare a nop (no op) submission.
Definition rtio.h:483
void rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len)
Release memory that was allocated by the RTIO's memory pool.
static int rtio_sqe_copy_in(struct rtio *r, const struct rtio_sqe *sqes, size_t sqe_count)
Copy an array of SQEs into the queue.
Definition rtio.h:1347
static void rtio_cqe_produce(struct rtio *r, struct rtio_cqe *cqe)
Produce a complete queue event if available.
Definition rtio.h:950
#define RTIO_OP_TINY_TX
An operation that transmits tiny writes by copying the data to write.
Definition rtio.h:466
static uint32_t rtio_cqe_compute_flags(struct rtio_iodev_sqe *iodev_sqe)
Compute the CQE flags from the rtio_iodev_sqe entry.
Definition rtio.h:1031
void rtio_executor_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
static int rtio_block_pool_alloc(struct rtio *r, size_t min_sz, size_t max_sz, uint8_t **buf, uint32_t *buf_len)
Definition rtio.h:669
int rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes, struct rtio_sqe **handle, size_t sqe_count)
Copy an array of SQEs into the queue and get resulting handles back.
static struct rtio_cqe * rtio_cqe_pool_alloc(struct rtio_cqe_pool *pool)
Definition rtio.h:645
struct k_mem_partition rtio_partition
The memory partition associated with all RTIO context information.
static void rtio_sqe_prep_read(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, uint8_t *buf, uint32_t len, void *userdata)
Prepare a read op submission.
Definition rtio.h:496
static struct rtio_sqe * rtio_sqe_acquire(struct rtio *r)
Acquire a single submission queue event if available.
Definition rtio.h:901
#define RTIO_OP_TX
An operation that transmits (writes)
Definition rtio.h:463
static void rtio_sqe_drop_all(struct rtio *r)
Drop all previously acquired sqe.
Definition rtio.h:919
static void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, void *userdata)
Definition rtio.h:525
int rtio_cqe_copy_out(struct rtio *r, struct rtio_cqe *cqes, size_t cqe_count, k_timeout_t timeout)
Copy an array of CQEs from the queue.
static void rtio_sqe_prep_callback(struct rtio_sqe *sqe, rtio_callback_t callback, void *arg0, void *userdata)
Prepare a callback op submission.
Definition rtio.h:588
static void rtio_access_grant(struct rtio *r, struct k_thread *t)
Grant access to an RTIO context to a user thread.
Definition rtio.h:1253
#define RTIO_OP_TXRX
An operation that transceives (reads and writes simultaneously)
Definition rtio.h:472
static void rtio_cqe_release(struct rtio *r, struct rtio_cqe *cqe)
Release consumed completion queue event.
Definition rtio.h:1020
static int rtio_sqe_rx_buf(const struct rtio_iodev_sqe *iodev_sqe, uint32_t min_buf_len, uint32_t max_buf_len, uint8_t **buf, uint32_t *buf_len)
Get the buffer associate with the RX submission.
Definition rtio.h:1179
static void rtio_iodev_sqe_err(struct rtio_iodev_sqe *iodev_sqe, int result)
Inform the executor of a submissions completion with error.
Definition rtio.h:1122
static void rtio_sqe_prep_write(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, uint8_t *buf, uint32_t len, void *userdata)
Prepare a write op submission.
Definition rtio.h:536
void(* rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, void *arg0)
Callback signature for RTIO_OP_CALLBACK.
Definition rtio.h:227
int rtio_sqe_cancel(struct rtio_sqe *sqe)
Attempt to cancel an SQE.
static void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
Definition rtio.h:638
static void rtio_iodev_sqe_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
Inform the executor of a submission completion with success.
Definition rtio.h:1109
#define RTIO_OP_NOP
An operation that does nothing and will complete immediately.
Definition rtio.h:457
static struct rtio_cqe * rtio_cqe_acquire(struct rtio *r)
Acquire a complete queue event if available.
Definition rtio.h:934
static struct rtio_iodev_sqe * rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the chain.
Definition rtio.h:871
static struct rtio_cqe * rtio_cqe_consume(struct rtio *r)
Consume a single completion queue event if available.
Definition rtio.h:966
static struct rtio_iodev_sqe * rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
Definition rtio.h:623
static struct rtio_iodev_sqe * rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the chain or transaction.
Definition rtio.h:888
int rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe, uint8_t **buff, uint32_t *buff_len)
Retrieve the mempool buffer that was allocated for the CQE.
static struct rtio_iodev_sqe * rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the transaction.
Definition rtio.h:853
void rtio_executor_submit(struct rtio *r)
static struct rtio_cqe * rtio_cqe_consume_block(struct rtio *r)
Wait for and consume a single completion queue event.
Definition rtio.h:996
static void rtio_block_pool_free(struct rtio *r, void *buf, uint32_t buf_len)
Definition rtio.h:706
#define RTIO_OP_RX
An operation that receives (reads)
Definition rtio.h:460
int rtio_submit(struct rtio *r, uint32_t wait_count)
Submit I/O requests to the underlying executor.
void k_sem_reset(struct k_sem *sem)
Resets a semaphore's count to zero.
void k_sem_give(struct k_sem *sem)
Give a semaphore.
int k_sem_take(struct k_sem *sem, k_timeout_t timeout)
Take a semaphore.
#define BIT(n)
Unsigned integer with bit position n set (signed in assembly language).
Definition util_macro.h:44
#define CONTAINER_OF(ptr, type, field)
Get a pointer to a structure containing the element.
Definition util.h:268
#define DIV_ROUND_UP(n, d)
Divide and round up.
Definition util.h:336
#define EINVAL
Invalid argument.
Definition errno.h:60
#define ENOMEM
Not enough core.
Definition errno.h:50
#define ENOTSUP
Unsupported value.
Definition errno.h:114
void k_yield(void)
Yield the current thread.
int32_t k_sleep(k_timeout_t timeout)
Put the current thread to sleep.
void k_object_access_grant(const void *object, struct k_thread *thread)
Grant a thread access to a kernel object.
Public kernel APIs.
struct k_thread t
Definition kobject.c:1324
struct k_queue q
Definition kobject.c:1320
Memory Blocks Allocator.
A wait-free intrusive multi producer single consumer (MPSC) queue using a singly linked list.
flags
Definition parser.h:96
void * ptr
Definition printk.c:177
__UINT32_TYPE__ uint32_t
Definition stdint.h:90
__INT32_TYPE__ int32_t
Definition stdint.h:74
__UINT8_TYPE__ uint8_t
Definition stdint.h:88
#define UINT16_MAX
Definition stdint.h:28
__UINTPTR_TYPE__ uintptr_t
Definition stdint.h:105
__UINT16_TYPE__ uint16_t
Definition stdint.h:89
__INT8_TYPE__ int8_t
Definition stdint.h:72
void * memset(void *buf, int c, size_t n)
void * memcpy(void *ZRESTRICT d, const void *ZRESTRICT s, size_t n)
Memory Partition.
Definition mem_domain.h:55
Thread Structure.
Definition thread.h:259
Kernel timeout type.
Definition sys_clock.h:65
Kernel timepoint type.
Definition sys_clock.h:219
Queue member.
Definition mpsc_lockfree.h:79
MPSC Queue.
Definition mpsc_lockfree.h:86
Definition errno.c:37
Definition rtio.h:309
struct rtio_cqe * pool
Definition rtio.h:313
struct mpsc free_q
Definition rtio.h:310
const uint16_t pool_size
Definition rtio.h:311
uint16_t pool_free
Definition rtio.h:312
A completion queue event.
Definition rtio.h:294
void * userdata
Associated userdata with operation.
Definition rtio.h:298
struct mpsc_node q
Definition rtio.h:295
uint32_t flags
Flags associated with the operation.
Definition rtio.h:299
int32_t result
Result from operation.
Definition rtio.h:297
API that an RTIO IO device should implement.
Definition rtio.h:433
void(* submit)(struct rtio_iodev_sqe *iodev_sqe)
Submit to the iodev an entry to work on.
Definition rtio.h:442
Compute the mempool block index for a given pointer.
Definition rtio.h:423
struct rtio_iodev_sqe * next
Definition rtio.h:426
struct rtio_sqe sqe
Definition rtio.h:424
struct rtio * r
Definition rtio.h:427
struct mpsc_node q
Definition rtio.h:425
An IO device with a function table for submitting requests.
Definition rtio.h:448
const struct rtio_iodev_api * api
Definition rtio.h:450
void * data
Definition rtio.h:453
Definition rtio.h:302
struct rtio_iodev_sqe * pool
Definition rtio.h:306
const uint16_t pool_size
Definition rtio.h:304
struct mpsc free_q
Definition rtio.h:303
uint16_t pool_free
Definition rtio.h:305
A submission queue event.
Definition rtio.h:232
uint32_t i2c_config
OP_I2C_CONFIGURE.
Definition rtio.h:282
void * userdata
User provided data which is returned upon operation completion.
Definition rtio.h:252
uint8_t * tx_buf
Definition rtio.h:277
uint8_t op
Op code.
Definition rtio.h:233
void * arg0
Last argument given to callback.
Definition rtio.h:271
uint8_t * rx_buf
Definition rtio.h:278
uint8_t prio
Op priority.
Definition rtio.h:235
uint8_t * buf
Buffer to use.
Definition rtio.h:259
uint32_t buf_len
Length of buffer.
Definition rtio.h:258
const struct rtio_iodev * iodev
Device to operation on.
Definition rtio.h:243
uint16_t flags
Op Flags.
Definition rtio.h:237
uint8_t tiny_buf_len
Length of tiny buffer.
Definition rtio.h:264
uint32_t txrx_buf_len
Definition rtio.h:276
uint8_t tiny_buf[7]
Tiny buffer.
Definition rtio.h:265
rtio_callback_t callback
Definition rtio.h:270
uint16_t iodev_flags
Op iodev flags.
Definition rtio.h:239
An RTIO context containing what can be viewed as a pair of queues.
Definition rtio.h:327
struct rtio_cqe_pool * cqe_pool
Definition rtio.h:357
struct mpsc sq
Definition rtio.h:365
atomic_t cq_count
Definition rtio.h:346
struct rtio_sqe_pool * sqe_pool
Definition rtio.h:354
atomic_t xcqcnt
Definition rtio.h:351
struct mpsc cq
Definition rtio.h:368
Misc utilities.