tests/amdgpu/hotunplug_tests.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22 */
  23
  24 #include <stdlib.h>
  25 #include <unistd.h>
  26 #include <sys/types.h>
  27 #include <sys/stat.h>
  28 #include <fcntl.h>
  29 #if HAVE_ALLOCA_H
  30 # include <alloca.h>
  31 #endif
  32
  33 #include "CUnit/Basic.h"
  34
  35 #include "amdgpu_test.h"
  36 #include "amdgpu_drm.h"
  37 #include "amdgpu_internal.h"
  38 #include "xf86drm.h"
  39 #include <pthread.h>
  40
  41 #define GFX_COMPUTE_NOP  0xffff1000
  42
  43 static  amdgpu_device_handle device_handle;
  44 static  uint32_t  major_version;
  45 static  uint32_t  minor_version;
  46 static char *sysfs_remove = NULL;
  47 static bool do_cs;
  48
  49 CU_BOOL suite_hotunplug_tests_enable(void)
  50 {
  51         CU_BOOL enable = CU_TRUE;
  52         drmDevicePtr device;
  53
  54         if (drmGetDevice2(drm_amdgpu[0], DRM_DEVICE_GET_PCI_REVISION, &device)) {
  55                 printf("\n\nGPU Failed to get DRM device PCI info!\n");
  56                 return CU_FALSE;
  57         }
  58
  59         if (device->bustype != DRM_BUS_PCI) {
  60                 printf("\n\nGPU device is not on PCI bus!\n");
  61                 amdgpu_device_deinitialize(device_handle);
  62                 return CU_FALSE;
  63         }
  64
  65         if (amdgpu_device_initialize(drm_amdgpu[0], &major_version,
  66                                              &minor_version, &device_handle))
  67                 return CU_FALSE;
  68
  69         /* Latest tested amdgpu version to work with all the tests */
  70         if (minor_version < 46)
  71                 enable = false;
  72
  73         /* skip hotplug test on APUs */
  74         if(device_handle->dev_info.ids_flags & AMDGPU_IDS_FLAGS_FUSION)
  75                 enable = false;
  76
  77         if (amdgpu_device_deinitialize(device_handle))
  78                 return CU_FALSE;
  79
  80         return enable;
  81 }
  82
  83 int suite_hotunplug_tests_init(void)
  84 {
  85         /* We need to open/close device at each test manually */
  86         amdgpu_close_devices();
  87
  88         return CUE_SUCCESS;
  89 }
  90
  91 int suite_hotunplug_tests_clean(void)
  92 {
  93
  94
  95         return CUE_SUCCESS;
  96 }
  97
  98 static int amdgpu_hotunplug_trigger(const char *pathname)
  99 {
 100         int fd, len;
 101
 102         fd = open(pathname, O_WRONLY);
 103         if (fd < 0)
 104                 return -errno;
 105
 106         len = write(fd, "1", 1);
 107         close(fd);
 108
 109         return len;
 110 }
 111
 112 static int amdgpu_hotunplug_setup_test()
 113 {
 114         int r;
 115         char *tmp_str;
 116
 117         if (amdgpu_open_device_on_test_index(open_render_node) < 0) {
 118                 printf("\n\n Failed to reopen device file!\n");
 119                 return CUE_SINIT_FAILED;
 120
 121
 122
 123         }
 124
 125         r = amdgpu_device_initialize(drm_amdgpu[0], &major_version,
 126                                    &minor_version, &device_handle);
 127
 128         if (r) {
 129                 if ((r == -EACCES) && (errno == EACCES))
 130                         printf("\n\nError:%s. "
 131                                 "Hint:Try to run this test program as root.",
 132                                 strerror(errno));
 133                 return CUE_SINIT_FAILED;
 134         }
 135
 136         tmp_str = amdgpu_get_device_from_fd(drm_amdgpu[0]);
 137         if (!tmp_str){
 138                 printf("\n\n Device path not found!\n");
 139                 return  CUE_SINIT_FAILED;
 140         }
 141
 142         sysfs_remove = realloc(tmp_str, strlen(tmp_str) * 2);
 143         strcat(sysfs_remove, "/remove");
 144
 145         return 0;
 146 }
 147
 148 static int amdgpu_hotunplug_teardown_test()
 149 {
 150         if (amdgpu_device_deinitialize(device_handle))
 151                 return CUE_SCLEAN_FAILED;
 152
 153         amdgpu_close_devices();
 154
 155         if (sysfs_remove)
 156                 free(sysfs_remove);
 157
 158         return 0;
 159 }
 160
 161 static inline int amdgpu_hotunplug_remove()
 162 {
 163         return amdgpu_hotunplug_trigger(sysfs_remove);
 164 }
 165
 166 static inline int amdgpu_hotunplug_rescan()
 167 {
 168         return amdgpu_hotunplug_trigger("/sys/bus/pci/rescan");
 169 }
 170
 171 static int amdgpu_cs_sync(amdgpu_context_handle context,
 172                            unsigned int ip_type,
 173                            int ring,
 174                            unsigned int seqno)
 175 {
 176         struct amdgpu_cs_fence fence = {
 177                 .context = context,
 178                 .ip_type = ip_type,
 179                 .ring = ring,
 180                 .fence = seqno,
 181         };
 182         uint32_t expired;
 183
 184         return  amdgpu_cs_query_fence_status(&fence,
 185                                            AMDGPU_TIMEOUT_INFINITE,
 186                                            0, &expired);
 187 }
 188
 189 static void *amdgpu_nop_cs()
 190 {
 191         amdgpu_bo_handle ib_result_handle;
 192         void *ib_result_cpu;
 193         uint64_t ib_result_mc_address;
 194         uint32_t *ptr;
 195         int i, r;
 196         amdgpu_bo_list_handle bo_list;
 197         amdgpu_va_handle va_handle;
 198         amdgpu_context_handle context;
 199         struct amdgpu_cs_request ibs_request;
 200         struct amdgpu_cs_ib_info ib_info;
 201
 202         r = amdgpu_cs_ctx_create(device_handle, &context);
 203         CU_ASSERT_EQUAL(r, 0);
 204
 205         r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096,
 206                                     AMDGPU_GEM_DOMAIN_GTT, 0,
 207                                     &ib_result_handle, &ib_result_cpu,
 208                                     &ib_result_mc_address, &va_handle);
 209         CU_ASSERT_EQUAL(r, 0);
 210
 211         ptr = ib_result_cpu;
 212         for (i = 0; i < 16; ++i)
 213                 ptr[i] = GFX_COMPUTE_NOP;
 214
 215         r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list);
 216         CU_ASSERT_EQUAL(r, 0);
 217
 218         memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info));
 219         ib_info.ib_mc_address = ib_result_mc_address;
 220         ib_info.size = 16;
 221
 222         memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request));
 223         ibs_request.ip_type = AMDGPU_HW_IP_GFX;
 224         ibs_request.ring = 0;
 225         ibs_request.number_of_ibs = 1;
 226         ibs_request.ibs = &ib_info;
 227         ibs_request.resources = bo_list;
 228
 229         while (do_cs)
 230                 amdgpu_cs_submit(context, 0, &ibs_request, 1);
 231
 232         amdgpu_cs_sync(context, AMDGPU_HW_IP_GFX, 0, ibs_request.seq_no);
 233         amdgpu_bo_list_destroy(bo_list);
 234         amdgpu_bo_unmap_and_free(ib_result_handle, va_handle,
 235                                  ib_result_mc_address, 4096);
 236
 237         amdgpu_cs_ctx_free(context);
 238
 239         return (void *)0;
 240 }
 241
 242 static pthread_t* amdgpu_create_cs_thread()
 243 {
 244         int r;
 245         pthread_t *thread = malloc(sizeof(*thread));
 246         if (!thread)
 247                 return NULL;
 248
 249         do_cs = true;
 250
 251         r = pthread_create(thread, NULL, amdgpu_nop_cs, NULL);
 252         CU_ASSERT_EQUAL(r, 0);
 253
 254         /* Give thread enough time to start*/
 255         usleep(100000);
 256         return thread;
 257 }
 258
 259 static void amdgpu_destroy_cs_thread(pthread_t *thread)
 260 {
 261         void *status;
 262
 263         do_cs = false;
 264
 265         pthread_join(*thread, &status);
 266         CU_ASSERT_EQUAL(status, 0);
 267
 268         free(thread);
 269 }
 270
 271
 272 static void amdgpu_hotunplug_test(bool with_cs)
 273 {
 274         int r;
 275         pthread_t *thread = NULL;
 276
 277         r = amdgpu_hotunplug_setup_test();
 278         CU_ASSERT_EQUAL(r , 0);
 279
 280         if (with_cs) {
 281                 thread = amdgpu_create_cs_thread();
 282                 CU_ASSERT_NOT_EQUAL(thread, NULL);
 283         }
 284
 285         r = amdgpu_hotunplug_remove();
 286         CU_ASSERT_EQUAL(r > 0, 1);
 287
 288         if (with_cs)
 289                 amdgpu_destroy_cs_thread(thread);
 290
 291         r = amdgpu_hotunplug_teardown_test();
 292         CU_ASSERT_EQUAL(r , 0);
 293
 294         r = amdgpu_hotunplug_rescan();
 295         CU_ASSERT_EQUAL(r > 0, 1);
 296 }
 297
 298 static void amdgpu_hotunplug_simple(void)
 299 {
 300         amdgpu_hotunplug_test(false);
 301 }
 302
 303 static void amdgpu_hotunplug_with_cs(void)
 304 {
 305         amdgpu_hotunplug_test(true);
 306 }
 307
 308 static void amdgpu_hotunplug_with_exported_bo(void)
 309 {
 310         int r;
 311         uint32_t dma_buf_fd;
 312         unsigned int *ptr;
 313         amdgpu_bo_handle bo_handle;
 314
 315         struct amdgpu_bo_alloc_request request = {
 316                 .alloc_size = 4096,
 317                 .phys_alignment = 4096,
 318                 .preferred_heap = AMDGPU_GEM_DOMAIN_GTT,
 319                 .flags = 0,
 320         };
 321
 322         r = amdgpu_hotunplug_setup_test();
 323         CU_ASSERT_EQUAL(r , 0);
 324
 325         amdgpu_bo_alloc(device_handle, &request, &bo_handle);
 326         CU_ASSERT_EQUAL(r, 0);
 327
 328         r = amdgpu_bo_export(bo_handle, amdgpu_bo_handle_type_dma_buf_fd, &dma_buf_fd);
 329         CU_ASSERT_EQUAL(r, 0);
 330
 331         ptr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, dma_buf_fd, 0);
 332         CU_ASSERT_NOT_EQUAL(ptr,  MAP_FAILED);
 333
 334         r = amdgpu_hotunplug_remove();
 335         CU_ASSERT_EQUAL(r > 0, 1);
 336
 337         amdgpu_bo_free(bo_handle);
 338
 339         r = amdgpu_hotunplug_teardown_test();
 340         CU_ASSERT_EQUAL(r , 0);
 341
 342         *ptr = 0xdeafbeef;
 343
 344         munmap(ptr, 4096);
 345         close (dma_buf_fd);
 346
 347         r = amdgpu_hotunplug_rescan();
 348         CU_ASSERT_EQUAL(r > 0, 1);
 349 }
 350
 351 static void amdgpu_hotunplug_with_exported_fence(void)
 352 {
 353         amdgpu_bo_handle ib_result_handle;
 354         void *ib_result_cpu;
 355         uint64_t ib_result_mc_address;
 356         uint32_t *ptr, sync_obj_handle, sync_obj_handle2;
 357         int i, r;
 358         amdgpu_bo_list_handle bo_list;
 359         amdgpu_va_handle va_handle;
 360         uint32_t major2, minor2;
 361         amdgpu_device_handle device2;
 362         amdgpu_context_handle context;
 363         struct amdgpu_cs_request ibs_request;
 364         struct amdgpu_cs_ib_info ib_info;
 365         struct amdgpu_cs_fence fence_status = {0};
 366         int shared_fd;
 367
 368         r = amdgpu_hotunplug_setup_test();
 369         CU_ASSERT_EQUAL(r , 0);
 370
 371         r = amdgpu_device_initialize(drm_amdgpu[1], &major2, &minor2, &device2);
 372         CU_ASSERT_EQUAL(r, 0);
 373
 374         r = amdgpu_cs_ctx_create(device_handle, &context);
 375         CU_ASSERT_EQUAL(r, 0);
 376
 377         r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096,
 378                                     AMDGPU_GEM_DOMAIN_GTT, 0,
 379                                     &ib_result_handle, &ib_result_cpu,
 380                                     &ib_result_mc_address, &va_handle);
 381         CU_ASSERT_EQUAL(r, 0);
 382
 383         ptr = ib_result_cpu;
 384         for (i = 0; i < 16; ++i)
 385                 ptr[i] = GFX_COMPUTE_NOP;
 386
 387         r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list);
 388         CU_ASSERT_EQUAL(r, 0);
 389
 390         memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info));
 391         ib_info.ib_mc_address = ib_result_mc_address;
 392         ib_info.size = 16;
 393
 394         memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request));
 395         ibs_request.ip_type = AMDGPU_HW_IP_GFX;
 396         ibs_request.ring = 0;
 397         ibs_request.number_of_ibs = 1;
 398         ibs_request.ibs = &ib_info;
 399         ibs_request.resources = bo_list;
 400
 401         CU_ASSERT_EQUAL(amdgpu_cs_submit(context, 0, &ibs_request, 1), 0);
 402
 403         fence_status.context = context;
 404         fence_status.ip_type = AMDGPU_HW_IP_GFX;
 405         fence_status.ip_instance = 0;
 406         fence_status.fence = ibs_request.seq_no;
 407
 408         CU_ASSERT_EQUAL(amdgpu_cs_fence_to_handle(device_handle, &fence_status,
 409                                                 AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ,
 410                                                 &sync_obj_handle),
 411                                                 0);
 412
 413         CU_ASSERT_EQUAL(amdgpu_cs_export_syncobj(device_handle, sync_obj_handle, &shared_fd), 0);
 414
 415         CU_ASSERT_EQUAL(amdgpu_cs_import_syncobj(device2, shared_fd, &sync_obj_handle2), 0);
 416
 417         CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device_handle, sync_obj_handle), 0);
 418
 419         CU_ASSERT_EQUAL(amdgpu_bo_list_destroy(bo_list), 0);
 420         CU_ASSERT_EQUAL(amdgpu_bo_unmap_and_free(ib_result_handle, va_handle,
 421                                  ib_result_mc_address, 4096), 0);
 422         CU_ASSERT_EQUAL(amdgpu_cs_ctx_free(context), 0);
 423
 424         r = amdgpu_hotunplug_remove();
 425         CU_ASSERT_EQUAL(r > 0, 1);
 426
 427         CU_ASSERT_EQUAL(amdgpu_cs_syncobj_wait(device2, &sync_obj_handle2, 1, 100000000, 0, NULL), 0);
 428
 429         CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device2, sync_obj_handle2), 0);
 430
 431         amdgpu_device_deinitialize(device2);
 432
 433         r = amdgpu_hotunplug_teardown_test();
 434         CU_ASSERT_EQUAL(r , 0);
 435
 436         r = amdgpu_hotunplug_rescan();
 437         CU_ASSERT_EQUAL(r > 0, 1);
 438 }
 439
 440
 441 CU_TestInfo hotunplug_tests[] = {
 442         { "Unplug card and rescan the bus to plug it back", amdgpu_hotunplug_simple },
 443         { "Same as first test but with command submission", amdgpu_hotunplug_with_cs },
 444         { "Unplug with exported bo", amdgpu_hotunplug_with_exported_bo },
 445         { "Unplug with exported fence", amdgpu_hotunplug_with_exported_fence },
 446         CU_TEST_INFO_NULL,
 447 };