From bc3854476f36d816d52cd8d41d1ecab2f8b6cdcf Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Fri, 24 Jan 2014 16:18:08 -0600 Subject: [PATCH] crypto: ccp - Use a single queue for proper ordering of tfm requests Move to a single queue to serialize requests within a tfm. When testing using IPSec with a large number of network connections the per cpu tfm queuing logic was not working properly. Signed-off-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-main.c | 164 ++++++++++------------------------- 1 file changed, 48 insertions(+), 116 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index b3f22b0..010fded 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); static LIST_HEAD(hash_algs); static LIST_HEAD(cipher_algs); -/* For any tfm, requests for that tfm on the same CPU must be returned - * in the order received. With multiple queues available, the CCP can - * process more than one cmd at a time. Therefore we must maintain - * a cmd list to insure the proper ordering of requests on a given tfm/cpu - * combination. +/* For any tfm, requests for that tfm must be returned on the order + * received. With multiple queues available, the CCP can process more + * than one cmd at a time. Therefore we must maintain a cmd list to insure + * the proper ordering of requests on a given tfm. */ -struct ccp_crypto_cpu_queue { +struct ccp_crypto_queue { struct list_head cmds; struct list_head *backlog; unsigned int cmd_count; }; -#define CCP_CRYPTO_MAX_QLEN 50 +#define CCP_CRYPTO_MAX_QLEN 100 -struct ccp_crypto_percpu_queue { - struct ccp_crypto_cpu_queue __percpu *cpu_queue; -}; -static struct ccp_crypto_percpu_queue req_queue; +static struct ccp_crypto_queue req_queue; +static spinlock_t req_queue_lock; struct ccp_crypto_cmd { struct list_head entry; @@ -71,8 +68,6 @@ struct ccp_crypto_cmd { /* Used for held command processing to determine state */ int ret; - - int cpu; }; struct ccp_crypto_cpu { @@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err) return true; } -/* - * ccp_crypto_cmd_complete must be called while running on the appropriate - * cpu and the caller must have done a get_cpu to disable preemption - */ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog) { - struct ccp_crypto_cpu_queue *cpu_queue; struct ccp_crypto_cmd *held = NULL, *tmp; + unsigned long flags; *backlog = NULL; - cpu_queue = this_cpu_ptr(req_queue.cpu_queue); + spin_lock_irqsave(&req_queue_lock, flags); /* Held cmds will be after the current cmd in the queue so start * searching for a cmd with a matching tfm for submission. */ tmp = crypto_cmd; - list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) { + list_for_each_entry_continue(tmp, &req_queue.cmds, entry) { if (crypto_cmd->tfm != tmp->tfm) continue; held = tmp; @@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( * Because cmds can be executed from any point in the cmd list * special precautions have to be taken when handling the backlog. */ - if (cpu_queue->backlog != &cpu_queue->cmds) { + if (req_queue.backlog != &req_queue.cmds) { /* Skip over this cmd if it is the next backlog cmd */ - if (cpu_queue->backlog == &crypto_cmd->entry) - cpu_queue->backlog = crypto_cmd->entry.next; + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; - *backlog = container_of(cpu_queue->backlog, + *backlog = container_of(req_queue.backlog, struct ccp_crypto_cmd, entry); - cpu_queue->backlog = cpu_queue->backlog->next; + req_queue.backlog = req_queue.backlog->next; /* Skip over this cmd if it is now the next backlog cmd */ - if (cpu_queue->backlog == &crypto_cmd->entry) - cpu_queue->backlog = crypto_cmd->entry.next; + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; } /* Remove the cmd entry from the list of cmds */ - cpu_queue->cmd_count--; + req_queue.cmd_count--; list_del(&crypto_cmd->entry); + spin_unlock_irqrestore(&req_queue_lock, flags); + return held; } -static void ccp_crypto_complete_on_cpu(struct work_struct *work) +static void ccp_crypto_complete(void *data, int err) { - struct ccp_crypto_cpu *cpu_work = - container_of(work, struct ccp_crypto_cpu, work); - struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd; + struct ccp_crypto_cmd *crypto_cmd = data; struct ccp_crypto_cmd *held, *next, *backlog; struct crypto_async_request *req = crypto_cmd->req; struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm); - int cpu, ret; - - cpu = get_cpu(); + int ret; - if (cpu_work->err == -EINPROGRESS) { + if (err == -EINPROGRESS) { /* Only propogate the -EINPROGRESS if necessary */ if (crypto_cmd->ret == -EBUSY) { crypto_cmd->ret = -EINPROGRESS; req->complete(req, -EINPROGRESS); } - goto e_cpu; + return; } /* Operation has completed - update the queue before invoking @@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) req->complete(req, -EINPROGRESS); /* Completion callbacks */ - ret = cpu_work->err; + ret = err; if (ctx->complete) ret = ctx->complete(req, ret); req->complete(req, ret); @@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) } kfree(crypto_cmd); - -e_cpu: - put_cpu(); - - complete(&cpu_work->completion); -} - -static void ccp_crypto_complete(void *data, int err) -{ - struct ccp_crypto_cmd *crypto_cmd = data; - struct ccp_crypto_cpu cpu_work; - - INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu); - init_completion(&cpu_work.completion); - cpu_work.crypto_cmd = crypto_cmd; - cpu_work.err = err; - - schedule_work_on(crypto_cmd->cpu, &cpu_work.work); - - /* Keep the completion call synchronous */ - wait_for_completion(&cpu_work.completion); } static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) { - struct ccp_crypto_cpu_queue *cpu_queue; struct ccp_crypto_cmd *active = NULL, *tmp; - int cpu, ret; - - cpu = get_cpu(); - crypto_cmd->cpu = cpu; + unsigned long flags; + int ret; - cpu_queue = this_cpu_ptr(req_queue.cpu_queue); + spin_lock_irqsave(&req_queue_lock, flags); /* Check if the cmd can/should be queued */ - if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { ret = -EBUSY; if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) - goto e_cpu; + goto e_lock; } /* Look for an entry with the same tfm. If there is a cmd - * with the same tfm in the list for this cpu then the current - * cmd cannot be submitted to the CCP yet. + * with the same tfm in the list then the current cmd cannot + * be submitted to the CCP yet. */ - list_for_each_entry(tmp, &cpu_queue->cmds, entry) { + list_for_each_entry(tmp, &req_queue.cmds, entry) { if (crypto_cmd->tfm != tmp->tfm) continue; active = tmp; @@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) if (!active) { ret = ccp_enqueue_cmd(crypto_cmd->cmd); if (!ccp_crypto_success(ret)) - goto e_cpu; + goto e_lock; } - if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { ret = -EBUSY; - if (cpu_queue->backlog == &cpu_queue->cmds) - cpu_queue->backlog = &crypto_cmd->entry; + if (req_queue.backlog == &req_queue.cmds) + req_queue.backlog = &crypto_cmd->entry; } crypto_cmd->ret = ret; - cpu_queue->cmd_count++; - list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds); + req_queue.cmd_count++; + list_add_tail(&crypto_cmd->entry, &req_queue.cmds); -e_cpu: - put_cpu(); +e_lock: + spin_unlock_irqrestore(&req_queue_lock, flags); return ret; } @@ -387,50 +352,18 @@ static void ccp_unregister_algs(void) } } -static int ccp_init_queues(void) -{ - struct ccp_crypto_cpu_queue *cpu_queue; - int cpu; - - req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue); - if (!req_queue.cpu_queue) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu); - INIT_LIST_HEAD(&cpu_queue->cmds); - cpu_queue->backlog = &cpu_queue->cmds; - cpu_queue->cmd_count = 0; - } - - return 0; -} - -static void ccp_fini_queue(void) -{ - struct ccp_crypto_cpu_queue *cpu_queue; - int cpu; - - for_each_possible_cpu(cpu) { - cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu); - BUG_ON(!list_empty(&cpu_queue->cmds)); - } - free_percpu(req_queue.cpu_queue); -} - static int ccp_crypto_init(void) { int ret; - ret = ccp_init_queues(); - if (ret) - return ret; + spin_lock_init(&req_queue_lock); + INIT_LIST_HEAD(&req_queue.cmds); + req_queue.backlog = &req_queue.cmds; + req_queue.cmd_count = 0; ret = ccp_register_algs(); - if (ret) { + if (ret) ccp_unregister_algs(); - ccp_fini_queue(); - } return ret; } @@ -438,7 +371,6 @@ static int ccp_crypto_init(void) static void ccp_crypto_exit(void) { ccp_unregister_algs(); - ccp_fini_queue(); } module_init(ccp_crypto_init); -- 2.7.4