diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 758e25893b380..90caf748fdeb5 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -39,8 +40,13 @@ #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 +/* + * Maximum handle value for static handles. + * Static handles are pre-defined, fixed numeric values statically assigned + * in the IDL file or FastRPC framework. + */ #define FASTRPC_MAX_STATIC_HANDLE (20) -#define FASTRPC_CTXID_MASK (0xFF00) +#define FASTRPC_CTXID_MASK GENMASK(15, 8) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" @@ -107,21 +113,12 @@ #define SENSORS_PD (2) #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) - /* Poll response number from remote processor for call completion */ #define FASTRPC_POLL_RESPONSE (0xdecaf) /* Polling mode timeout limit */ #define FASTRPC_POLL_MAX_TIMEOUT_US (10000) -/* Response types supported for RPC calls */ -enum fastrpc_response_flags { - /* normal job completion glink response */ - NORMAL_RESPONSE = 0, - /* process updates poll memory instead of glink response */ - POLL_MODE = 1, -}; - struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */ @@ -253,13 +250,13 @@ struct fastrpc_invoke_ctx { u64 *fdlist; u32 *crc; /* Poll memory that DSP updates */ - u32 *poll; + u32 *poll_addr; u64 ctxid; u64 msg_sz; /* work done status flag */ bool is_work_done; - /* response flags from remote processor */ - enum fastrpc_response_flags rsp_flags; + /* process updates poll memory instead of glink response */ + bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; @@ -302,6 +299,7 @@ struct fastrpc_channel_ctx { struct fastrpc_buf *remote_heap; bool secure; bool unsigned_support; + bool poll_mode_supported; u64 dma_mask; }; @@ -325,6 +323,8 @@ struct fastrpc_user { int client_id; int pd; bool is_secure_dev; + /* Flags poll mode state */ + bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ @@ -564,7 +564,7 @@ static void fastrpc_context_free(struct kref *ref) fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); - idr_remove(&cctx->ctx_idr, ctx->ctxid >> 8); + idr_remove(&cctx->ctx_idr, FIELD_GET(FASTRPC_CTXID_MASK, ctx->ctxid)); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); @@ -734,7 +734,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } - ctx->ctxid = ret << 8; + ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret); spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); @@ -1100,8 +1100,9 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); ctx->fdlist = (u64 *)(pages + ctx->nscalars); - ctx->crc = (u32 *)(ctx->fdlist + FASTRPC_MAX_FDLIST); - ctx->poll = (u32 *)(ctx->crc + FASTRPC_MAX_CRCLIST); + ctx->poll_addr = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST + + sizeof(u32) * FASTRPC_MAX_CRCLIST); + args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra; @@ -1204,6 +1205,7 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, union fastrpc_remote_arg *rpra = ctx->rpra; struct fastrpc_user *fl = ctx->fl; struct fastrpc_map *mmap = NULL; + u64 *fdlist = ctx->fdlist; int i, inbufs; int ret = 0; @@ -1229,9 +1231,9 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { - if (!ctx->fdlist[i]) + if (!fdlist[i]) break; - if (!fastrpc_map_lookup(fl, (int)ctx->fdlist[i], &mmap)) + if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap)) fastrpc_map_put(mmap); } @@ -1270,12 +1272,10 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, } -static inline u32 fastrpc_poll_op(void *p) +static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx) { - struct fastrpc_invoke_ctx *ctx = p; - dma_rmb(); - return READ_ONCE(*ctx->poll); + return READ_ONCE(*ctx->poll_addr); } static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) @@ -1284,15 +1284,18 @@ static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) int ret; /* - * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll + * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr * or until another path marks the work done. */ - ret = read_poll_timeout_atomic(fastrpc_poll_op, val, - (val == FASTRPC_POLL_RESPONSE) || - ctx->is_work_done, 1, + ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val, + (val == FASTRPC_POLL_RESPONSE) || ctx->is_work_done, 1, FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx); if (!ret && val == FASTRPC_POLL_RESPONSE) { + /* + * DSP writes FASTRPC_POLL_RESPONSE to signal successful + * completion via the poll path. + */ ctx->is_work_done = true; ctx->retval = 0; } @@ -1323,28 +1326,15 @@ static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, { int err; - do { - switch (ctx->rsp_flags) { - case NORMAL_RESPONSE: - err = fastrpc_wait_for_response(ctx, kernel); - if (err || ctx->is_work_done) - return err; - break; - case POLL_MODE: - err = poll_for_remote_response(ctx); - /* If polling timed out, move to normal response mode */ - if (err) - ctx->rsp_flags = NORMAL_RESPONSE; - break; - default: - err = -EBADR; - dev_dbg(ctx->fl->sctx->dev, - "unsupported response type:0x%x\n", ctx->rsp_flags); - break; - } - } while (!ctx->is_work_done); + if (ctx->is_polled) { + err = poll_for_remote_response(ctx); + if (!err) + return 0; + /* If polling timed out or failed, move to normal response mode */ + ctx->is_polled = false; + } - return err; + return fastrpc_wait_for_response(ctx, kernel); } static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, @@ -1388,20 +1378,18 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, if (err) goto bail; - if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD) - ctx->rsp_flags = POLL_MODE; - wait: + /* + * Set message context as polled if the call is for a user PD + * dynamic module and user has enabled poll mode. + */ + if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode) + ctx->is_polled = true; err = fastrpc_wait_for_completion(ctx, kernel); + if (err) goto bail; - if (!ctx->is_work_done) { - err = -ETIMEDOUT; - dev_dbg(fl->sctx->dev, "Invalid workdone state for handle 0x%x, sc 0x%x\n", - handle, sc); - goto bail; - } /* make sure that all memory writes by DSP are seen by CPU */ dma_rmb(); /* populate all the output buffers with results */ @@ -1931,6 +1919,35 @@ static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, return 0; } +static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_ioctl_set_option opt = {0}; + int i; + + if (copy_from_user(&opt, argp, sizeof(opt))) + return -EFAULT; + + for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) { + if (opt.reserved[i] != 0) + return -EINVAL; + } + + if (opt.request_id != FASTRPC_POLL_MODE) + return -EINVAL; + + if (!fl->cctx->poll_mode_supported) + return -EOPNOTSUPP; + + if (opt.value == FASTRPC_POLL_MODE_ENABLE) + fl->poll_mode = true; + else if (opt.value == FASTRPC_POLL_MODE_DISABLE) + fl->poll_mode = false; + else + return -EINVAL; + + return 0; +} + static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp) { struct fastrpc_ioctl_capability cap = {0}; @@ -2288,6 +2305,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MEM_MAP: err = fastrpc_req_mem_map(fl, argp); break; + case FASTRPC_IOCTL_SET_OPTION: + err = fastrpc_set_option(fl, argp); + break; case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; @@ -2437,6 +2457,21 @@ static int fastrpc_get_domain_id(const char *domain) return -EINVAL; } +/* + * Exception list for older platforms that use default_soc_data but whose + * DSP firmware supports FastRPC polling mode. + * + * NOTE: This list is intentionally closed. + * Do NOT add new platforms here. New SoCs must advertise polling mode + * support via their soc_data. + */ + +static const char *const fastrpc_poll_supported_machines[] = { + "qcom,milos", "qcom,qcs8300", "qcom,sa8775p", "qcom,sar2130p", + "qcom,sm8450", "qcom,sm8550", "qcom,sm8650", "qcom,sm8750", + "qcom,x1e80100", "qcom,x1p42100", NULL, +}; + static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) { struct device *rdev = &rpdev->dev; @@ -2508,6 +2543,9 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); data->secure = secure_dsp; + data->poll_mode_supported = + of_machine_compatible_match(fastrpc_poll_supported_machines); + switch (domain_id) { case ADSP_DOMAIN_ID: case MDSP_DOMAIN_ID: @@ -2633,7 +2671,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, if (len < sizeof(*rsp)) return -EINVAL; - ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 8); + ctxid = FIELD_GET(FASTRPC_CTXID_MASK, rsp->ctx); spin_lock_irqsave(&cctx->lock, flags); ctx = idr_find(&cctx->ctx_idr, ctxid); @@ -2645,8 +2683,8 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, } ctx->retval = rsp->retval; - complete(&ctx->work); ctx->is_work_done = true; + complete(&ctx->work); /* * The DMA buffer associated with the context cannot be freed in diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index c6e2925f47e69..ba1ea5ed426c4 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -16,6 +16,7 @@ #define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static) #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) +#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option) #define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability) /** @@ -67,6 +68,28 @@ enum fastrpc_proc_attr { /* Fastrpc attribute for memory protection of buffers */ #define FASTRPC_ATTR_SECUREMAP (1) +/** + * FASTRPC_POLL_MODE - Enable/disable poll mode for FastRPC invocations + * + * Poll mode is an optimization that allows the CPU to poll shared memory + * for completion instead of waiting for an interrupt-based response. + * This reduces latency for fast-completing operations. + * + * Restrictions: + * - Only supported for USER_PD (User Protection Domain) + * - Only applies to dynamic modules (handle > 20) + * - Static modules always use interrupt-based completion + * + * Values: + * - 0: Disable poll mode (use interrupt-based completion) + * - 1: Enable poll mode (poll shared memory for completion) + */ +#define FASTRPC_POLL_MODE (1) + +/* Values for FASTRPC_POLL_MODE request */ +#define FASTRPC_POLL_MODE_DISABLE 0 +#define FASTRPC_POLL_MODE_ENABLE 1 + struct fastrpc_invoke_args { __u64 ptr; __u64 length; @@ -133,6 +156,12 @@ struct fastrpc_mem_unmap { __s32 reserved[5]; }; +struct fastrpc_ioctl_set_option { + __u32 request_id; /* Request type (e.g., FASTRPC_POLL_MODE) */ + __u32 value; /* Request-specific value */ + __s32 reserved[6]; +}; + struct fastrpc_ioctl_capability { __u32 unused; /* deprecated, ignored by the kernel */ __u32 attribute_id;