iommu/vt-d: Refactor prq_event_thread()
authorLu Baolu <baolu.lu@linux.intel.com>
Thu, 10 Jun 2021 02:01:01 +0000 (10:01 +0800)
committerJoerg Roedel <jroedel@suse.de>
Thu, 10 Jun 2021 07:06:13 +0000 (09:06 +0200)
Refactor prq_event_thread() by moving handling single prq event out of
the main loop.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Link: https://lore.kernel.org/r/20210520031531.712333-1-baolu.lu@linux.intel.com
Link: https://lore.kernel.org/r/20210610020115.1637656-10-baolu.lu@linux.intel.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
drivers/iommu/intel/svm.c

index 57867ff..d51ddec 100644 (file)
@@ -866,141 +866,174 @@ intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc)
        return iommu_report_device_fault(dev, &event);
 }
 
+static void handle_bad_prq_event(struct intel_iommu *iommu,
+                                struct page_req_dsc *req, int result)
+{
+       struct qi_desc desc;
+
+       pr_err("%s: Invalid page request: %08llx %08llx\n",
+              iommu->name, ((unsigned long long *)req)[0],
+              ((unsigned long long *)req)[1]);
+
+       /*
+        * Per VT-d spec. v3.0 ch7.7, system software must
+        * respond with page group response if private data
+        * is present (PDP) or last page in group (LPIG) bit
+        * is set. This is an additional VT-d feature beyond
+        * PCI ATS spec.
+        */
+       if (!req->lpig && !req->priv_data_present)
+               return;
+
+       desc.qw0 = QI_PGRP_PASID(req->pasid) |
+                       QI_PGRP_DID(req->rid) |
+                       QI_PGRP_PASID_P(req->pasid_present) |
+                       QI_PGRP_PDP(req->priv_data_present) |
+                       QI_PGRP_RESP_CODE(result) |
+                       QI_PGRP_RESP_TYPE;
+       desc.qw1 = QI_PGRP_IDX(req->prg_index) |
+                       QI_PGRP_LPIG(req->lpig);
+       desc.qw2 = 0;
+       desc.qw3 = 0;
+
+       if (req->priv_data_present)
+               memcpy(&desc.qw2, req->priv_data, sizeof(req->priv_data));
+       qi_submit_sync(iommu, &desc, 1, 0);
+}
+
+static void handle_single_prq_event(struct intel_iommu *iommu,
+                                   struct mm_struct *mm,
+                                   struct page_req_dsc *req)
+{
+       u64 address = (u64)req->addr << VTD_PAGE_SHIFT;
+       int result = QI_RESP_INVALID;
+       struct vm_area_struct *vma;
+       struct qi_desc desc;
+       unsigned int flags;
+       vm_fault_t ret;
+
+       /* If the mm is already defunct, don't handle faults. */
+       if (!mmget_not_zero(mm))
+               goto response;
+
+       mmap_read_lock(mm);
+       vma = find_extend_vma(mm, address);
+       if (!vma || address < vma->vm_start)
+               goto invalid;
+
+       if (access_error(vma, req))
+               goto invalid;
+
+       flags = FAULT_FLAG_USER | FAULT_FLAG_REMOTE;
+       if (req->wr_req)
+               flags |= FAULT_FLAG_WRITE;
+
+       ret = handle_mm_fault(vma, address, flags, NULL);
+       if (!(ret & VM_FAULT_ERROR))
+               result = QI_RESP_SUCCESS;
+invalid:
+       mmap_read_unlock(mm);
+       mmput(mm);
+
+response:
+       if (!(req->lpig || req->priv_data_present))
+               return;
+
+       desc.qw0 = QI_PGRP_PASID(req->pasid) |
+                       QI_PGRP_DID(req->rid) |
+                       QI_PGRP_PASID_P(req->pasid_present) |
+                       QI_PGRP_PDP(req->priv_data_present) |
+                       QI_PGRP_RESP_CODE(result) |
+                       QI_PGRP_RESP_TYPE;
+       desc.qw1 = QI_PGRP_IDX(req->prg_index) |
+                       QI_PGRP_LPIG(req->lpig);
+       desc.qw2 = 0;
+       desc.qw3 = 0;
+
+       if (req->priv_data_present)
+               memcpy(&desc.qw2, req->priv_data, sizeof(req->priv_data));
+
+       qi_submit_sync(iommu, &desc, 1, 0);
+}
+
 static irqreturn_t prq_event_thread(int irq, void *d)
 {
        struct intel_svm_dev *sdev = NULL;
        struct intel_iommu *iommu = d;
        struct intel_svm *svm = NULL;
-       int head, tail, handled = 0;
-       unsigned int flags = 0;
+       struct page_req_dsc *req;
+       int head, tail, handled;
+       u64 address;
 
-       /* Clear PPR bit before reading head/tail registers, to
-        * ensure that we get a new interrupt if needed. */
+       /*
+        * Clear PPR bit before reading head/tail registers, to ensure that
+        * we get a new interrupt if needed.
+        */
        writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);
 
        tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
        head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
+       handled = (head != tail);
        while (head != tail) {
-               struct vm_area_struct *vma;
-               struct page_req_dsc *req;
-               struct qi_desc resp;
-               int result;
-               vm_fault_t ret;
-               u64 address;
-
-               handled = 1;
                req = &iommu->prq[head / sizeof(*req)];
-               result = QI_RESP_INVALID;
                address = (u64)req->addr << VTD_PAGE_SHIFT;
-               if (!req->pasid_present) {
-                       pr_err("%s: Page request without PASID: %08llx %08llx\n",
-                              iommu->name, ((unsigned long long *)req)[0],
-                              ((unsigned long long *)req)[1]);
-                       goto no_pasid;
+
+               if (unlikely(!req->pasid_present)) {
+                       pr_err("IOMMU: %s: Page request without PASID\n",
+                              iommu->name);
+bad_req:
+                       svm = NULL;
+                       sdev = NULL;
+                       handle_bad_prq_event(iommu, req, QI_RESP_INVALID);
+                       goto prq_advance;
                }
-               /* We shall not receive page request for supervisor SVM */
-               if (req->pm_req && (req->rd_req | req->wr_req)) {
-                       pr_err("Unexpected page request in Privilege Mode");
-                       /* No need to find the matching sdev as for bad_req */
-                       goto no_pasid;
+
+               if (unlikely(!is_canonical_address(address))) {
+                       pr_err("IOMMU: %s: Address is not canonical\n",
+                              iommu->name);
+                       goto bad_req;
+               }
+
+               if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) {
+                       pr_err("IOMMU: %s: Page request in Privilege Mode\n",
+                              iommu->name);
+                       goto bad_req;
                }
-               /* DMA read with exec requeset is not supported. */
-               if (req->exe_req && req->rd_req) {
-                       pr_err("Execution request not supported\n");
-                       goto no_pasid;
+
+               if (unlikely(req->exe_req && req->rd_req)) {
+                       pr_err("IOMMU: %s: Execution request not supported\n",
+                              iommu->name);
+                       goto bad_req;
                }
+
                if (!svm || svm->pasid != req->pasid) {
-                       rcu_read_lock();
-                       svm = pasid_private_find(req->pasid);
-                       /* It *can't* go away, because the driver is not permitted
+                       /*
+                        * It can't go away, because the driver is not permitted
                         * to unbind the mm while any page faults are outstanding.
-                        * So we only need RCU to protect the internal idr code. */
-                       rcu_read_unlock();
-                       if (IS_ERR_OR_NULL(svm)) {
-                               pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
-                                      iommu->name, req->pasid, ((unsigned long long *)req)[0],
-                                      ((unsigned long long *)req)[1]);
-                               goto no_pasid;
-                       }
+                        */
+                       svm = pasid_private_find(req->pasid);
+                       if (IS_ERR_OR_NULL(svm) || (svm->flags & SVM_FLAG_SUPERVISOR_MODE))
+                               goto bad_req;
                }
 
-               if (!sdev || sdev->sid != req->rid)
+               if (!sdev || sdev->sid != req->rid) {
                        sdev = svm_lookup_device_by_sid(svm, req->rid);
-
-               /* Since we're using init_mm.pgd directly, we should never take
-                * any faults on kernel addresses. */
-               if (!svm->mm)
-                       goto bad_req;
-
-               /* If address is not canonical, return invalid response */
-               if (!is_canonical_address(address))
-                       goto bad_req;
+                       if (!sdev)
+                               goto bad_req;
+               }
 
                /*
                 * If prq is to be handled outside iommu driver via receiver of
                 * the fault notifiers, we skip the page response here.
                 */
                if (svm->flags & SVM_FLAG_GUEST_MODE) {
-                       if (sdev && !intel_svm_prq_report(sdev->dev, req))
+                       if (!intel_svm_prq_report(sdev->dev, req))
                                goto prq_advance;
                        else
                                goto bad_req;
                }
 
-               /* If the mm is already defunct, don't handle faults. */
-               if (!mmget_not_zero(svm->mm))
-                       goto bad_req;
-
-               mmap_read_lock(svm->mm);
-               vma = find_extend_vma(svm->mm, address);
-               if (!vma || address < vma->vm_start)
-                       goto invalid;
-
-               if (access_error(vma, req))
-                       goto invalid;
-
-               flags = FAULT_FLAG_USER | FAULT_FLAG_REMOTE;
-               if (req->wr_req)
-                       flags |= FAULT_FLAG_WRITE;
-
-               ret = handle_mm_fault(vma, address, flags, NULL);
-               if (ret & VM_FAULT_ERROR)
-                       goto invalid;
-
-               result = QI_RESP_SUCCESS;
-invalid:
-               mmap_read_unlock(svm->mm);
-               mmput(svm->mm);
-bad_req:
-               /* We get here in the error case where the PASID lookup failed,
-                  and these can be NULL. Do not use them below this point! */
-               sdev = NULL;
-               svm = NULL;
-no_pasid:
-               if (req->lpig || req->priv_data_present) {
-                       /*
-                        * Per VT-d spec. v3.0 ch7.7, system software must
-                        * respond with page group response if private data
-                        * is present (PDP) or last page in group (LPIG) bit
-                        * is set. This is an additional VT-d feature beyond
-                        * PCI ATS spec.
-                        */
-                       resp.qw0 = QI_PGRP_PASID(req->pasid) |
-                               QI_PGRP_DID(req->rid) |
-                               QI_PGRP_PASID_P(req->pasid_present) |
-                               QI_PGRP_PDP(req->priv_data_present) |
-                               QI_PGRP_RESP_CODE(result) |
-                               QI_PGRP_RESP_TYPE;
-                       resp.qw1 = QI_PGRP_IDX(req->prg_index) |
-                               QI_PGRP_LPIG(req->lpig);
-                       resp.qw2 = 0;
-                       resp.qw3 = 0;
-
-                       if (req->priv_data_present)
-                               memcpy(&resp.qw2, req->priv_data,
-                                      sizeof(req->priv_data));
-                       qi_submit_sync(iommu, &resp, 1, 0);
-               }
+               handle_single_prq_event(iommu, svm->mm, req);
 prq_advance:
                head = (head + sizeof(*req)) & PRQ_RING_MASK;
        }