#include <linux/hwmon.h>
#include <uapi/misc/habanalabs.h>
-#define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
-
enum hl_device_status hl_device_status(struct hl_device *hdev)
{
enum hl_device_status status;
cdev_device_del(&hdev->cdev, hdev->dev);
}
+static void device_hard_reset_pending(struct work_struct *work)
+{
+ struct hl_device_reset_work *device_reset_work =
+ container_of(work, struct hl_device_reset_work,
+ reset_work.work);
+ struct hl_device *hdev = device_reset_work->hdev;
+ int rc;
+
+ rc = hl_device_reset(hdev, true, true);
+ if ((rc == -EBUSY) && !hdev->device_fini_pending) {
+ dev_info(hdev->dev,
+ "Could not reset device. will try again in %u seconds",
+ HL_PENDING_RESET_PER_SEC);
+
+ queue_delayed_work(device_reset_work->wq,
+ &device_reset_work->reset_work,
+ msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
+ }
+}
+
/*
* device_early_init - do some early initialization for the habanalabs device
*
hl_cb_mgr_init(&hdev->kernel_cb_mgr);
+ hdev->device_reset_work.wq =
+ create_singlethread_workqueue("hl_device_reset");
+ if (!hdev->device_reset_work.wq) {
+ rc = -ENOMEM;
+ dev_err(hdev->dev, "Failed to create device reset WQ\n");
+ goto free_cb_mgr;
+ }
+
+ INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work,
+ device_hard_reset_pending);
+ hdev->device_reset_work.hdev = hdev;
+ hdev->device_fini_pending = 0;
+
mutex_init(&hdev->send_cpu_message_lock);
mutex_init(&hdev->debug_lock);
mutex_init(&hdev->mmu_cache_lock);
return 0;
+free_cb_mgr:
+ hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
free_idle_busy_ts_arr:
kfree(hdev->idle_busy_ts_arr);
free_chip_info:
kfree(hdev->hl_chip_info);
destroy_workqueue(hdev->eq_wq);
+ destroy_workqueue(hdev->device_reset_work.wq);
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
destroy_workqueue(hdev->cq_wq[i]);
return rc;
}
-static int device_kill_open_processes(struct hl_device *hdev)
+static int device_kill_open_processes(struct hl_device *hdev, u32 timeout)
{
- u16 pending_total, pending_cnt;
struct hl_fpriv *hpriv;
struct task_struct *task = NULL;
+ u32 pending_cnt;
- if (hdev->pldm)
- pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
- else
- pending_total = HL_PENDING_RESET_PER_SEC;
/* Giving time for user to close FD, and for processes that are inside
* hl_device_open to finish
if (!list_empty(&hdev->fpriv_list))
ssleep(1);
+ if (timeout) {
+ pending_cnt = timeout;
+ } else {
+ if (hdev->process_kill_trial_cnt) {
+ /* Processes have been already killed */
+ pending_cnt = 1;
+ goto wait_for_processes;
+ } else {
+ /* Wait a small period after process kill */
+ pending_cnt = HL_PENDING_RESET_PER_SEC;
+ }
+ }
+
mutex_lock(&hdev->fpriv_list_lock);
/* This section must be protected because we are dereferencing
* continuing with the reset.
*/
- pending_cnt = pending_total;
-
+wait_for_processes:
while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
- dev_info(hdev->dev,
- "Waiting for all user contexts to get closed before hard reset\n");
+ dev_dbg(hdev->dev,
+ "Waiting for all unmap operations to finish before hard reset\n");
pending_cnt--;
ssleep(1);
}
- return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
-}
+ /* All processes exited successfully */
+ if (list_empty(&hdev->fpriv_list))
+ return 0;
-static void device_hard_reset_pending(struct work_struct *work)
-{
- struct hl_device_reset_work *device_reset_work =
- container_of(work, struct hl_device_reset_work, reset_work);
- struct hl_device *hdev = device_reset_work->hdev;
+ /* Give up waiting for processes to exit */
+ if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
+ return -ETIME;
- hl_device_reset(hdev, true, true);
+ hdev->process_kill_trial_cnt++;
- kfree(device_reset_work);
+ return -EBUSY;
}
/*
hard_reset = true;
}
+ /* Re-entry of reset thread */
+ if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
+ goto kill_processes;
+
/*
* Prevent concurrency in this function - only one reset should be
* done at any given time. Only need to perform this if we didn't
again:
if ((hard_reset) && (!from_hard_reset_thread)) {
- struct hl_device_reset_work *device_reset_work;
-
hdev->hard_reset_pending = true;
- device_reset_work = kzalloc(sizeof(*device_reset_work),
- GFP_ATOMIC);
- if (!device_reset_work) {
- rc = -ENOMEM;
- goto out_err;
- }
+ hdev->process_kill_trial_cnt = 0;
/*
* Because the reset function can't run from interrupt or
* from heartbeat work, we need to call the reset function
* from a dedicated work
*/
- INIT_WORK(&device_reset_work->reset_work,
- device_hard_reset_pending);
- device_reset_work->hdev = hdev;
- schedule_work(&device_reset_work->reset_work);
+ queue_delayed_work(hdev->device_reset_work.wq,
+ &hdev->device_reset_work.reset_work, 0);
return 0;
}
/* Go over all the queues, release all CS and their jobs */
hl_cs_rollback_all(hdev);
+kill_processes:
if (hard_reset) {
/* Kill processes here after CS rollback. This is because the
* process can't really exit until all its CSs are done, which
* is what we do in cs rollback
*/
- rc = device_kill_open_processes(hdev);
+ rc = device_kill_open_processes(hdev, 0);
+
+ if (rc == -EBUSY) {
+ if (hdev->device_fini_pending) {
+ dev_crit(hdev->dev,
+ "Failed to kill all open processes, stopping hard reset\n");
+ goto out_err;
+ }
+
+ /* signal reset thread to reschedule */
+ return rc;
+ }
+
if (rc) {
dev_crit(hdev->dev,
"Failed to kill all open processes, stopping hard reset\n");
*/
void hl_device_fini(struct hl_device *hdev)
{
- int i, rc;
ktime_t timeout;
+ int i, rc;
dev_info(hdev->dev, "Removing device\n");
+ hdev->device_fini_pending = 1;
+ flush_delayed_work(&hdev->device_reset_work.reset_work);
+
/*
* This function is competing with the reset function, so try to
* take the reset atomic and if we are already in middle of reset,
* can't really exit until all its CSs are done, which is what we
* do in cs rollback
*/
- rc = device_kill_open_processes(hdev);
+ dev_info(hdev->dev,
+ "Waiting for all processes to exit (timeout of %u seconds)",
+ HL_PENDING_RESET_LONG_SEC);
+
+ rc = device_kill_open_processes(hdev, HL_PENDING_RESET_LONG_SEC);
if (rc)
dev_crit(hdev->dev, "Failed to kill all open processes\n");
#define HL_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFull >> PAGE_SHIFT)
#define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK)
-#define HL_PENDING_RESET_PER_SEC 30
+#define HL_PENDING_RESET_PER_SEC 10
+#define HL_PENDING_RESET_MAX_TRIALS 60 /* 10 minutes */
+#define HL_PENDING_RESET_LONG_SEC 60
#define HL_HARD_RESET_MAX_TIMEOUT 120
/**
* struct hl_device_reset_work - reset workqueue task wrapper.
+ * @wq: work queue for device reset procedure.
* @reset_work: reset work to be done.
* @hdev: habanalabs device structure.
*/
struct hl_device_reset_work {
- struct work_struct reset_work;
+ struct workqueue_struct *wq;
+ struct delayed_work reset_work;
struct hl_device *hdev;
};
* @dev_ctrl: related kernel device structure for the control device
* @work_freq: delayed work to lower device frequency if possible.
* @work_heartbeat: delayed work for CPU-CP is-alive check.
+ * @device_reset_work: delayed work which performs hard reset
* @asic_name: ASIC specific name.
* @asic_type: ASIC specific type.
* @completion_queue: array of hl_cq.
* @supports_cb_mapping: is mapping a CB to the device's MMU supported.
* @needs_reset: true if reset_on_lockup is false and device should be reset
* due to lockup.
+ * @process_kill_trial_cnt: number of trials reset thread tried killing
+ * user processes
+ * @device_fini_pending: true if device_fini was called and might be
+ * waiting for the reset thread to finish
*/
struct hl_device {
struct pci_dev *pdev;
struct device *dev_ctrl;
struct delayed_work work_freq;
struct delayed_work work_heartbeat;
+ struct hl_device_reset_work device_reset_work;
char asic_name[HL_STR_MAX];
char status[HL_DEV_STS_MAX][HL_STR_MAX];
enum hl_asic_type asic_type;
u8 supports_soft_reset;
u8 supports_cb_mapping;
u8 needs_reset;
+ u8 process_kill_trial_cnt;
+ u8 device_fini_pending;
/* Parameters for bring-up */
u64 nic_ports_mask;