multipathd/main.c

   1 /*
   2  * Copyright (c) 2004, 2005 Christophe Varoqui
   3  * Copyright (c) 2005 Kiyoshi Ueda, NEC
   4  * Copyright (c) 2005 Benjamin Marzinski, Redhat
   5  * Copyright (c) 2005 Edward Goggin, EMC
   6  */
   7 #include <unistd.h>
   8 #include <sys/stat.h>
   9 #include <libdevmapper.h>
  10 #include <wait.h>
  11 #include <sys/mman.h>
  12 #include <sys/types.h>
  13 #include <fcntl.h>
  14 #include <errno.h>
  15 #include <sys/time.h>
  16 #include <sys/resource.h>
  17 #include <limits.h>
  18 #include <linux/oom.h>
  19 #include <libudev.h>
  20 #ifdef USE_SYSTEMD
  21 #include <systemd/sd-daemon.h>
  22 #endif
  23 #include <semaphore.h>
  24 #include <mpath_persist.h>
  25 #include <time.h>
  26
  27 /*
  28  * libcheckers
  29  */
  30 #include <checkers.h>
  31
  32 /*
  33  * libmultipath
  34  */
  35 #include <parser.h>
  36 #include <vector.h>
  37 #include <memory.h>
  38 #include <config.h>
  39 #include <util.h>
  40 #include <hwtable.h>
  41 #include <defaults.h>
  42 #include <structs.h>
  43 #include <blacklist.h>
  44 #include <structs_vec.h>
  45 #include <dmparser.h>
  46 #include <devmapper.h>
  47 #include <sysfs.h>
  48 #include <dict.h>
  49 #include <discovery.h>
  50 #include <debug.h>
  51 #include <propsel.h>
  52 #include <uevent.h>
  53 #include <switchgroup.h>
  54 #include <print.h>
  55 #include <configure.h>
  56 #include <prio.h>
  57 #include <pgpolicies.h>
  58 #include <uevent.h>
  59 #include <log.h>
  60
  61 #include "main.h"
  62 #include "pidfile.h"
  63 #include "uxlsnr.h"
  64 #include "uxclnt.h"
  65 #include "cli.h"
  66 #include "cli_handlers.h"
  67 #include "lock.h"
  68 #include "waiter.h"
  69 #include "wwids.h"
  70
  71 #define FILE_NAME_SIZE 256
  72 #define CMDSIZE 160
  73
  74 #define LOG_MSG(a, b) \
  75 do { \
  76         if (pp->offline) \
  77                 condlog(a, "%s: %s - path offline", pp->mpp->alias, pp->dev); \
  78         else if (strlen(b)) \
  79                 condlog(a, "%s: %s - %s", pp->mpp->alias, pp->dev, b); \
  80 } while(0)
  81
  82 struct mpath_event_param
  83 {
  84         char * devname;
  85         struct multipath *mpp;
  86 };
  87
  88 unsigned int mpath_mx_alloc_len;
  89
  90 int logsink;
  91 enum daemon_status running_state;
  92 pid_t daemon_pid;
  93
  94 static sem_t exit_sem;
  95 /*
  96  * global copy of vecs for use in sig handlers
  97  */
  98 struct vectors * gvecs;
  99
 100 struct udev * udev;
 101
 102 static int
 103 need_switch_pathgroup (struct multipath * mpp, int refresh)
 104 {
 105         struct pathgroup * pgp;
 106         struct path * pp;
 107         unsigned int i, j;
 108
 109         if (!mpp || mpp->pgfailback == -FAILBACK_MANUAL)
 110                 return 0;
 111
 112         /*
 113          * Refresh path priority values
 114          */
 115         if (refresh)
 116                 vector_foreach_slot (mpp->pg, pgp, i)
 117                         vector_foreach_slot (pgp->paths, pp, j)
 118                                 pathinfo(pp, conf->hwtable, DI_PRIO);
 119
 120         mpp->bestpg = select_path_group(mpp);
 121
 122         if (mpp->bestpg != mpp->nextpg)
 123                 return 1;
 124
 125         return 0;
 126 }
 127
 128 static void
 129 switch_pathgroup (struct multipath * mpp)
 130 {
 131         mpp->stat_switchgroup++;
 132         dm_switchgroup(mpp->alias, mpp->bestpg);
 133         condlog(2, "%s: switch to path group #%i",
 134                  mpp->alias, mpp->bestpg);
 135 }
 136
 137 static int
 138 coalesce_maps(struct vectors *vecs, vector nmpv)
 139 {
 140         struct multipath * ompp;
 141         vector ompv = vecs->mpvec;
 142         unsigned int i;
 143
 144         vector_foreach_slot (ompv, ompp, i) {
 145                 condlog(3, "%s: coalesce map", ompp->alias);
 146                 if (!find_mp_by_wwid(nmpv, ompp->wwid)) {
 147                         /*
 148                          * remove all current maps not allowed by the
 149                          * current configuration
 150                          */
 151                         if (dm_flush_map(ompp->alias)) {
 152                                 condlog(0, "%s: unable to flush devmap",
 153                                         ompp->alias);
 154                                 /*
 155                                  * may be just because the device is open
 156                                  */
 157                                 if (setup_multipath(vecs, ompp) != 0) {
 158                                         i--;
 159                                         continue;
 160                                 }
 161                                 if (!vector_alloc_slot(nmpv))
 162                                         return 1;
 163
 164                                 vector_set_slot(nmpv, ompp);
 165
 166                                 vector_del_slot(ompv, i);
 167                                 i--;
 168                         }
 169                         else {
 170                                 dm_lib_release();
 171                                 condlog(2, "%s devmap removed", ompp->alias);
 172                         }
 173                 } else if (conf->reassign_maps) {
 174                         condlog(3, "%s: Reassign existing device-mapper"
 175                                 " devices", ompp->alias);
 176                         dm_reassign(ompp->alias);
 177                 }
 178         }
 179         return 0;
 180 }
 181
 182 void
 183 sync_map_state(struct multipath *mpp)
 184 {
 185         struct pathgroup *pgp;
 186         struct path *pp;
 187         unsigned int i, j;
 188
 189         if (!mpp->pg)
 190                 return;
 191
 192         vector_foreach_slot (mpp->pg, pgp, i){
 193                 vector_foreach_slot (pgp->paths, pp, j){
 194                         if (pp->state == PATH_UNCHECKED ||
 195                             pp->state == PATH_WILD)
 196                                 continue;
 197                         if ((pp->dmstate == PSTATE_FAILED ||
 198                              pp->dmstate == PSTATE_UNDEF) &&
 199                             (pp->state == PATH_UP || pp->state == PATH_GHOST))
 200                                 dm_reinstate_path(mpp->alias, pp->dev_t);
 201                         else if ((pp->dmstate == PSTATE_ACTIVE ||
 202                                   pp->dmstate == PSTATE_UNDEF) &&
 203                                  (pp->state == PATH_DOWN ||
 204                                   pp->state == PATH_SHAKY))
 205                                 dm_fail_path(mpp->alias, pp->dev_t);
 206                 }
 207         }
 208 }
 209
 210 static void
 211 sync_maps_state(vector mpvec)
 212 {
 213         unsigned int i;
 214         struct multipath *mpp;
 215
 216         vector_foreach_slot (mpvec, mpp, i)
 217                 sync_map_state(mpp);
 218 }
 219
 220 static int
 221 flush_map(struct multipath * mpp, struct vectors * vecs)
 222 {
 223         /*
 224          * clear references to this map before flushing so we can ignore
 225          * the spurious uevent we may generate with the dm_flush_map call below
 226          */
 227         if (dm_flush_map(mpp->alias)) {
 228                 /*
 229                  * May not really be an error -- if the map was already flushed
 230                  * from the device mapper by dmsetup(8) for instance.
 231                  */
 232                 condlog(0, "%s: can't flush", mpp->alias);
 233                 return 1;
 234         }
 235         else {
 236                 dm_lib_release();
 237                 condlog(2, "%s: map flushed", mpp->alias);
 238         }
 239
 240         orphan_paths(vecs->pathvec, mpp);
 241         remove_map_and_stop_waiter(mpp, vecs, 1);
 242
 243         return 0;
 244 }
 245
 246 static int
 247 uev_add_map (struct uevent * uev, struct vectors * vecs)
 248 {
 249         char *alias;
 250         int major = -1, minor = -1, rc;
 251
 252         condlog(3, "%s: add map (uevent)", uev->kernel);
 253         alias = uevent_get_dm_name(uev);
 254         if (!alias) {
 255                 condlog(3, "%s: No DM_NAME in uevent", uev->kernel);
 256                 major = uevent_get_major(uev);
 257                 minor = uevent_get_minor(uev);
 258                 alias = dm_mapname(major, minor);
 259                 if (!alias) {
 260                         condlog(2, "%s: mapname not found for %d:%d",
 261                                 uev->kernel, major, minor);
 262                         return 1;
 263                 }
 264         }
 265         rc = ev_add_map(uev->kernel, alias, vecs);
 266         FREE(alias);
 267         return rc;
 268 }
 269
 270 int
 271 ev_add_map (char * dev, char * alias, struct vectors * vecs)
 272 {
 273         char * refwwid;
 274         struct multipath * mpp;
 275         int map_present;
 276         int r = 1;
 277
 278         map_present = dm_map_present(alias);
 279
 280         if (map_present && dm_type(alias, TGT_MPATH) <= 0) {
 281                 condlog(4, "%s: not a multipath map", alias);
 282                 return 0;
 283         }
 284
 285         mpp = find_mp_by_alias(vecs->mpvec, alias);
 286
 287         if (mpp) {
 288                 /*
 289                  * Not really an error -- we generate our own uevent
 290                  * if we create a multipath mapped device as a result
 291                  * of uev_add_path
 292                  */
 293                 if (conf->reassign_maps) {
 294                         condlog(3, "%s: Reassign existing device-mapper devices",
 295                                 alias);
 296                         dm_reassign(alias);
 297                 }
 298                 return 0;
 299         }
 300         condlog(2, "%s: adding map", alias);
 301
 302         /*
 303          * now we can register the map
 304          */
 305         if (map_present && (mpp = add_map_without_path(vecs, alias))) {
 306                 sync_map_state(mpp);
 307                 condlog(2, "%s: devmap %s registered", alias, dev);
 308                 return 0;
 309         }
 310         r = get_refwwid(dev, DEV_DEVMAP, vecs->pathvec, &refwwid);
 311
 312         if (refwwid) {
 313                 r = coalesce_paths(vecs, NULL, refwwid, 0);
 314                 dm_lib_release();
 315         }
 316
 317         if (!r)
 318                 condlog(2, "%s: devmap %s added", alias, dev);
 319         else if (r == 2)
 320                 condlog(2, "%s: uev_add_map %s blacklisted", alias, dev);
 321         else
 322                 condlog(0, "%s: uev_add_map %s failed", alias, dev);
 323
 324         FREE(refwwid);
 325         return r;
 326 }
 327
 328 static int
 329 uev_remove_map (struct uevent * uev, struct vectors * vecs)
 330 {
 331         char *alias;
 332         int minor;
 333         struct multipath *mpp;
 334
 335         condlog(2, "%s: remove map (uevent)", uev->kernel);
 336         alias = uevent_get_dm_name(uev);
 337         if (!alias) {
 338                 condlog(3, "%s: No DM_NAME in uevent, ignoring", uev->kernel);
 339                 return 0;
 340         }
 341         minor = uevent_get_minor(uev);
 342         mpp = find_mp_by_minor(vecs->mpvec, minor);
 343
 344         if (!mpp) {
 345                 condlog(2, "%s: devmap not registered, can't remove",
 346                         uev->kernel);
 347                 goto out;
 348         }
 349         if (strcmp(mpp->alias, alias)) {
 350                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
 351                         mpp->alias, mpp->dmi->minor, minor);
 352                 goto out;
 353         }
 354
 355         orphan_paths(vecs->pathvec, mpp);
 356         remove_map_and_stop_waiter(mpp, vecs, 1);
 357 out:
 358         FREE(alias);
 359         return 0;
 360 }
 361
 362 int
 363 ev_remove_map (char * devname, char * alias, int minor, struct vectors * vecs)
 364 {
 365         struct multipath * mpp;
 366
 367         mpp = find_mp_by_minor(vecs->mpvec, minor);
 368
 369         if (!mpp) {
 370                 condlog(2, "%s: devmap not registered, can't remove",
 371                         devname);
 372                 return 0;
 373         }
 374         if (strcmp(mpp->alias, alias)) {
 375                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
 376                         mpp->alias, mpp->dmi->minor, minor);
 377                 return 0;
 378         }
 379         return flush_map(mpp, vecs);
 380 }
 381
 382 static int
 383 uev_add_path (struct uevent *uev, struct vectors * vecs)
 384 {
 385         struct path *pp;
 386         int ret, i;
 387
 388         condlog(2, "%s: add path (uevent)", uev->kernel);
 389         if (strstr(uev->kernel, "..") != NULL) {
 390                 /*
 391                  * Don't allow relative device names in the pathvec
 392                  */
 393                 condlog(0, "%s: path name is invalid", uev->kernel);
 394                 return 1;
 395         }
 396
 397         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
 398         if (pp) {
 399                 condlog(0, "%s: spurious uevent, path already in pathvec",
 400                         uev->kernel);
 401                 if (pp->mpp)
 402                         return 0;
 403                 if (!strlen(pp->wwid)) {
 404                         udev_device_unref(pp->udev);
 405                         pp->udev = udev_device_ref(uev->udev);
 406                         ret = pathinfo(pp, conf->hwtable,
 407                                        DI_ALL | DI_BLACKLIST);
 408                         if (ret == 2) {
 409                                 i = find_slot(vecs->pathvec, (void *)pp);
 410                                 if (i != -1)
 411                                         vector_del_slot(vecs->pathvec, i);
 412                                 free_path(pp);
 413                                 return 0;
 414                         } else if (ret == 1) {
 415                                 condlog(0, "%s: failed to reinitialize path",
 416                                         uev->kernel);
 417                                 return 1;
 418                         }
 419                 }
 420         } else {
 421                 /*
 422                  * get path vital state
 423                  */
 424                 ret = store_pathinfo(vecs->pathvec, conf->hwtable,
 425                                      uev->udev, DI_ALL, &pp);
 426                 if (!pp) {
 427                         if (ret == 2)
 428                                 return 0;
 429                         condlog(0, "%s: failed to store path info",
 430                                 uev->kernel);
 431                         return 1;
 432                 }
 433                 pp->checkint = conf->checkint;
 434         }
 435
 436         return ev_add_path(pp, vecs);
 437 }
 438
 439 /*
 440  * returns:
 441  * 0: added
 442  * 1: error
 443  */
 444 int
 445 ev_add_path (struct path * pp, struct vectors * vecs)
 446 {
 447         struct multipath * mpp;
 448         char empty_buff[WWID_SIZE] = {0};
 449         char params[PARAMS_SIZE] = {0};
 450         int retries = 3;
 451         int start_waiter = 0;
 452
 453         /*
 454          * need path UID to go any further
 455          */
 456         if (memcmp(empty_buff, pp->wwid, WWID_SIZE) == 0) {
 457                 condlog(0, "%s: failed to get path uid", pp->dev);
 458                 goto fail; /* leave path added to pathvec */
 459         }
 460         mpp = pp->mpp = find_mp_by_wwid(vecs->mpvec, pp->wwid);
 461 rescan:
 462         if (mpp) {
 463                 if ((!pp->size) || (mpp->size != pp->size)) {
 464                         if (!pp->size)
 465                                 condlog(0, "%s: failed to add new path %s, "
 466                                         "device size is 0",
 467                                         mpp->alias, pp->dev);
 468                         else
 469                                 condlog(0, "%s: failed to add new path %s, "
 470                                         "device size mismatch",
 471                                         mpp->alias, pp->dev);
 472                         int i = find_slot(vecs->pathvec, (void *)pp);
 473                         if (i != -1)
 474                                 vector_del_slot(vecs->pathvec, i);
 475                         free_path(pp);
 476                         return 1;
 477                 }
 478
 479                 condlog(4,"%s: adopting all paths for path %s",
 480                         mpp->alias, pp->dev);
 481                 if (adopt_paths(vecs->pathvec, mpp, 1))
 482                         goto fail; /* leave path added to pathvec */
 483
 484                 verify_paths(mpp, vecs, NULL);
 485                 mpp->flush_on_last_del = FLUSH_UNDEF;
 486                 mpp->action = ACT_RELOAD;
 487         }
 488         else {
 489                 if (!pp->size) {
 490                         condlog(0, "%s: failed to create new map,"
 491                                 " device size is 0 ", pp->dev);
 492                         int i = find_slot(vecs->pathvec, (void *)pp);
 493                         if (i != -1)
 494                                 vector_del_slot(vecs->pathvec, i);
 495                         free_path(pp);
 496                         return 1;
 497                 }
 498
 499                 condlog(4,"%s: creating new map", pp->dev);
 500                 if ((mpp = add_map_with_path(vecs, pp, 1))) {
 501                         mpp->action = ACT_CREATE;
 502                         /*
 503                          * We don't depend on ACT_CREATE, as domap will
 504                          * set it to ACT_NOTHING when complete.
 505                          */
 506                         start_waiter = 1;
 507                 }
 508                 else
 509                         goto fail; /* leave path added to pathvec */
 510         }
 511
 512         /* persistent reseravtion check*/
 513         mpath_pr_event_handle(pp);
 514
 515         /*
 516          * push the map to the device-mapper
 517          */
 518         if (setup_map(mpp, params, PARAMS_SIZE)) {
 519                 condlog(0, "%s: failed to setup map for addition of new "
 520                         "path %s", mpp->alias, pp->dev);
 521                 goto fail_map;
 522         }
 523         /*
 524          * reload the map for the multipath mapped device
 525          */
 526         if (domap(mpp, params) <= 0) {
 527                 condlog(0, "%s: failed in domap for addition of new "
 528                         "path %s", mpp->alias, pp->dev);
 529                 /*
 530                  * deal with asynchronous uevents :((
 531                  */
 532                 if (mpp->action == ACT_RELOAD && retries-- > 0) {
 533                         condlog(0, "%s: uev_add_path sleep", mpp->alias);
 534                         sleep(1);
 535                         update_mpp_paths(mpp, vecs->pathvec);
 536                         goto rescan;
 537                 }
 538                 else if (mpp->action == ACT_RELOAD)
 539                         condlog(0, "%s: giving up reload", mpp->alias);
 540                 else
 541                         goto fail_map;
 542         }
 543         dm_lib_release();
 544
 545         /*
 546          * update our state from kernel regardless of create or reload
 547          */
 548         if (setup_multipath(vecs, mpp))
 549                 goto fail; /* if setup_multipath fails, it removes the map */
 550
 551         sync_map_state(mpp);
 552
 553         if ((mpp->action == ACT_CREATE ||
 554              (mpp->action == ACT_NOTHING && start_waiter && !mpp->waiter)) &&
 555             start_waiter_thread(mpp, vecs))
 556                         goto fail_map;
 557
 558         if (retries >= 0) {
 559                 condlog(2, "%s [%s]: path added to devmap %s",
 560                         pp->dev, pp->dev_t, mpp->alias);
 561                 return 0;
 562         }
 563         else
 564                 return 1;
 565
 566 fail_map:
 567         remove_map(mpp, vecs, 1);
 568 fail:
 569         orphan_path(pp, "failed to add path");
 570         return 1;
 571 }
 572
 573 static int
 574 uev_remove_path (struct uevent *uev, struct vectors * vecs)
 575 {
 576         struct path *pp;
 577
 578         condlog(2, "%s: remove path (uevent)", uev->kernel);
 579         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
 580
 581         if (!pp) {
 582                 /* Not an error; path might have been purged earlier */
 583                 condlog(0, "%s: path already removed", uev->kernel);
 584                 return 0;
 585         }
 586
 587         return ev_remove_path(pp, vecs);
 588 }
 589
 590 int
 591 ev_remove_path (struct path *pp, struct vectors * vecs)
 592 {
 593         struct multipath * mpp;
 594         int i, retval = 0;
 595         char params[PARAMS_SIZE] = {0};
 596
 597         /*
 598          * avoid referring to the map of an orphaned path
 599          */
 600         if ((mpp = pp->mpp)) {
 601                 /*
 602                  * transform the mp->pg vector of vectors of paths
 603                  * into a mp->params string to feed the device-mapper
 604                  */
 605                 if (update_mpp_paths(mpp, vecs->pathvec)) {
 606                         condlog(0, "%s: failed to update paths",
 607                                 mpp->alias);
 608                         goto fail;
 609                 }
 610                 if ((i = find_slot(mpp->paths, (void *)pp)) != -1)
 611                         vector_del_slot(mpp->paths, i);
 612
 613                 /*
 614                  * remove the map IFF removing the last path
 615                  */
 616                 if (VECTOR_SIZE(mpp->paths) == 0) {
 617                         char alias[WWID_SIZE];
 618
 619                         /*
 620                          * flush_map will fail if the device is open
 621                          */
 622                         strncpy(alias, mpp->alias, WWID_SIZE);
 623                         if (mpp->flush_on_last_del == FLUSH_ENABLED) {
 624                                 condlog(2, "%s Last path deleted, disabling queueing", mpp->alias);
 625                                 mpp->retry_tick = 0;
 626                                 mpp->no_path_retry = NO_PATH_RETRY_FAIL;
 627                                 mpp->flush_on_last_del = FLUSH_IN_PROGRESS;
 628                                 dm_queue_if_no_path(mpp->alias, 0);
 629                         }
 630                         if (!flush_map(mpp, vecs)) {
 631                                 condlog(2, "%s: removed map after"
 632                                         " removing all paths",
 633                                         alias);
 634                                 retval = 0;
 635                                 goto out;
 636                         }
 637                         /*
 638                          * Not an error, continue
 639                          */
 640                 }
 641
 642                 if (setup_map(mpp, params, PARAMS_SIZE)) {
 643                         condlog(0, "%s: failed to setup map for"
 644                                 " removal of path %s", mpp->alias, pp->dev);
 645                         goto fail;
 646                 }
 647                 /*
 648                  * reload the map
 649                  */
 650                 mpp->action = ACT_RELOAD;
 651                 if (domap(mpp, params) <= 0) {
 652                         condlog(0, "%s: failed in domap for "
 653                                 "removal of path %s",
 654                                 mpp->alias, pp->dev);
 655                         retval = 1;
 656                 } else {
 657                         /*
 658                          * update our state from kernel
 659                          */
 660                         if (setup_multipath(vecs, mpp)) {
 661                                 goto fail;
 662                         }
 663                         sync_map_state(mpp);
 664
 665                         condlog(2, "%s [%s]: path removed from map %s",
 666                                 pp->dev, pp->dev_t, mpp->alias);
 667                 }
 668         }
 669
 670 out:
 671         if ((i = find_slot(vecs->pathvec, (void *)pp)) != -1)
 672                 vector_del_slot(vecs->pathvec, i);
 673
 674         free_path(pp);
 675
 676         return retval;
 677
 678 fail:
 679         remove_map_and_stop_waiter(mpp, vecs, 1);
 680         return 1;
 681 }
 682
 683 static int
 684 uev_update_path (struct uevent *uev, struct vectors * vecs)
 685 {
 686         int ro, retval = 0;
 687
 688         ro = uevent_get_disk_ro(uev);
 689
 690         if (ro >= 0) {
 691                 struct path * pp;
 692
 693                 condlog(2, "%s: update path write_protect to '%d' (uevent)",
 694                         uev->kernel, ro);
 695                 pp = find_path_by_dev(vecs->pathvec, uev->kernel);
 696                 if (!pp) {
 697                         condlog(0, "%s: spurious uevent, path not found",
 698                                 uev->kernel);
 699                         return 1;
 700                 }
 701                 if (pp->mpp) {
 702                         retval = reload_map(vecs, pp->mpp, 0);
 703
 704                         condlog(2, "%s: map %s reloaded (retval %d)",
 705                                 uev->kernel, pp->mpp->alias, retval);
 706                 }
 707
 708         }
 709
 710         return retval;
 711 }
 712
 713 static int
 714 map_discovery (struct vectors * vecs)
 715 {
 716         struct multipath * mpp;
 717         unsigned int i;
 718
 719         if (dm_get_maps(vecs->mpvec))
 720                 return 1;
 721
 722         vector_foreach_slot (vecs->mpvec, mpp, i)
 723                 if (setup_multipath(vecs, mpp))
 724                         return 1;
 725
 726         return 0;
 727 }
 728
 729 int
 730 uxsock_trigger (char * str, char ** reply, int * len, void * trigger_data)
 731 {
 732         struct vectors * vecs;
 733         int r;
 734
 735         *reply = NULL;
 736         *len = 0;
 737         vecs = (struct vectors *)trigger_data;
 738
 739         pthread_cleanup_push(cleanup_lock, &vecs->lock);
 740         lock(vecs->lock);
 741         pthread_testcancel();
 742
 743         r = parse_cmd(str, reply, len, vecs);
 744
 745         if (r > 0) {
 746                 *reply = STRDUP("fail\n");
 747                 *len = strlen(*reply) + 1;
 748                 r = 1;
 749         }
 750         else if (!r && *len == 0) {
 751                 *reply = STRDUP("ok\n");
 752                 *len = strlen(*reply) + 1;
 753                 r = 0;
 754         }
 755         /* else if (r < 0) leave *reply alone */
 756
 757         lock_cleanup_pop(vecs->lock);
 758         return r;
 759 }
 760
 761 static int
 762 uev_discard(char * devpath)
 763 {
 764         char *tmp;
 765         char a[11], b[11];
 766
 767         /*
 768          * keep only block devices, discard partitions
 769          */
 770         tmp = strstr(devpath, "/block/");
 771         if (tmp == NULL){
 772                 condlog(4, "no /block/ in '%s'", devpath);
 773                 return 1;
 774         }
 775         if (sscanf(tmp, "/block/%10s", a) != 1 ||
 776             sscanf(tmp, "/block/%10[^/]/%10s", a, b) == 2) {
 777                 condlog(4, "discard event on %s", devpath);
 778                 return 1;
 779         }
 780         return 0;
 781 }
 782
 783 int
 784 uev_trigger (struct uevent * uev, void * trigger_data)
 785 {
 786         int r = 0;
 787         struct vectors * vecs;
 788
 789         vecs = (struct vectors *)trigger_data;
 790
 791         if (uev_discard(uev->devpath))
 792                 return 0;
 793
 794         pthread_cleanup_push(cleanup_lock, &vecs->lock);
 795         lock(vecs->lock);
 796         pthread_testcancel();
 797
 798         /*
 799          * device map event
 800          * Add events are ignored here as the tables
 801          * are not fully initialised then.
 802          */
 803         if (!strncmp(uev->kernel, "dm-", 3)) {
 804                 if (!strncmp(uev->action, "change", 6)) {
 805                         r = uev_add_map(uev, vecs);
 806                         goto out;
 807                 }
 808                 if (!strncmp(uev->action, "remove", 6)) {
 809                         r = uev_remove_map(uev, vecs);
 810                         goto out;
 811                 }
 812                 goto out;
 813         }
 814
 815         /*
 816          * path add/remove event
 817          */
 818         if (filter_devnode(conf->blist_devnode, conf->elist_devnode,
 819                            uev->kernel) > 0)
 820                 goto out;
 821
 822         if (!strncmp(uev->action, "add", 3)) {
 823                 r = uev_add_path(uev, vecs);
 824                 goto out;
 825         }
 826         if (!strncmp(uev->action, "remove", 6)) {
 827                 r = uev_remove_path(uev, vecs);
 828                 goto out;
 829         }
 830         if (!strncmp(uev->action, "change", 6)) {
 831                 r = uev_update_path(uev, vecs);
 832                 goto out;
 833         }
 834
 835 out:
 836         lock_cleanup_pop(vecs->lock);
 837         return r;
 838 }
 839
 840 static void *
 841 ueventloop (void * ap)
 842 {
 843         struct udev *udev = ap;
 844
 845         if (uevent_listen(udev))
 846                 condlog(0, "error starting uevent listener");
 847
 848         return NULL;
 849 }
 850
 851 static void *
 852 uevqloop (void * ap)
 853 {
 854         if (uevent_dispatch(&uev_trigger, ap))
 855                 condlog(0, "error starting uevent dispatcher");
 856
 857         return NULL;
 858 }
 859 static void *
 860 uxlsnrloop (void * ap)
 861 {
 862         if (cli_init())
 863                 return NULL;
 864
 865         set_handler_callback(LIST+PATHS, cli_list_paths);
 866         set_handler_callback(LIST+PATHS+FMT, cli_list_paths_fmt);
 867         set_handler_callback(LIST+MAPS, cli_list_maps);
 868         set_handler_callback(LIST+STATUS, cli_list_status);
 869         set_handler_callback(LIST+DAEMON, cli_list_daemon);
 870         set_handler_callback(LIST+MAPS+STATUS, cli_list_maps_status);
 871         set_handler_callback(LIST+MAPS+STATS, cli_list_maps_stats);
 872         set_handler_callback(LIST+MAPS+FMT, cli_list_maps_fmt);
 873         set_handler_callback(LIST+MAPS+TOPOLOGY, cli_list_maps_topology);
 874         set_handler_callback(LIST+TOPOLOGY, cli_list_maps_topology);
 875         set_handler_callback(LIST+MAP+TOPOLOGY, cli_list_map_topology);
 876         set_handler_callback(LIST+CONFIG, cli_list_config);
 877         set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
 878         set_handler_callback(LIST+DEVICES, cli_list_devices);
 879         set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
 880         set_handler_callback(ADD+PATH, cli_add_path);
 881         set_handler_callback(DEL+PATH, cli_del_path);
 882         set_handler_callback(ADD+MAP, cli_add_map);
 883         set_handler_callback(DEL+MAP, cli_del_map);
 884         set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
 885         set_handler_callback(RECONFIGURE, cli_reconfigure);
 886         set_handler_callback(SUSPEND+MAP, cli_suspend);
 887         set_handler_callback(RESUME+MAP, cli_resume);
 888         set_handler_callback(RESIZE+MAP, cli_resize);
 889         set_handler_callback(RELOAD+MAP, cli_reload);
 890         set_handler_callback(RESET+MAP, cli_reassign);
 891         set_handler_callback(REINSTATE+PATH, cli_reinstate);
 892         set_handler_callback(FAIL+PATH, cli_fail);
 893         set_handler_callback(DISABLEQ+MAP, cli_disable_queueing);
 894         set_handler_callback(RESTOREQ+MAP, cli_restore_queueing);
 895         set_handler_callback(DISABLEQ+MAPS, cli_disable_all_queueing);
 896         set_handler_callback(RESTOREQ+MAPS, cli_restore_all_queueing);
 897         set_handler_callback(QUIT, cli_quit);
 898         set_handler_callback(SHUTDOWN, cli_shutdown);
 899         set_handler_callback(GETPRSTATUS+MAP, cli_getprstatus);
 900         set_handler_callback(SETPRSTATUS+MAP, cli_setprstatus);
 901         set_handler_callback(UNSETPRSTATUS+MAP, cli_unsetprstatus);
 902         set_handler_callback(FORCEQ+DAEMON, cli_force_no_daemon_q);
 903         set_handler_callback(RESTOREQ+DAEMON, cli_restore_no_daemon_q);
 904
 905         umask(077);
 906         uxsock_listen(&uxsock_trigger, ap);
 907
 908         return NULL;
 909 }
 910
 911 void
 912 exit_daemon (void)
 913 {
 914         sem_post(&exit_sem);
 915 }
 916
 917 const char *
 918 daemon_status(void)
 919 {
 920         switch (running_state) {
 921         case DAEMON_INIT:
 922                 return "init";
 923         case DAEMON_START:
 924                 return "startup";
 925         case DAEMON_CONFIGURE:
 926                 return "configure";
 927         case DAEMON_RUNNING:
 928                 return "running";
 929         case DAEMON_SHUTDOWN:
 930                 return "shutdown";
 931         }
 932         return NULL;
 933 }
 934
 935 static void
 936 fail_path (struct path * pp, int del_active)
 937 {
 938         if (!pp->mpp)
 939                 return;
 940
 941         condlog(2, "checker failed path %s in map %s",
 942                  pp->dev_t, pp->mpp->alias);
 943
 944         dm_fail_path(pp->mpp->alias, pp->dev_t);
 945         if (del_active)
 946                 update_queue_mode_del_path(pp->mpp);
 947 }
 948
 949 /*
 950  * caller must have locked the path list before calling that function
 951  */
 952 static void
 953 reinstate_path (struct path * pp, int add_active)
 954 {
 955         if (!pp->mpp)
 956                 return;
 957
 958         if (dm_reinstate_path(pp->mpp->alias, pp->dev_t))
 959                 condlog(0, "%s: reinstate failed", pp->dev_t);
 960         else {
 961                 condlog(2, "%s: reinstated", pp->dev_t);
 962                 if (add_active)
 963                         update_queue_mode_add_path(pp->mpp);
 964         }
 965 }
 966
 967 static void
 968 enable_group(struct path * pp)
 969 {
 970         struct pathgroup * pgp;
 971
 972         /*
 973          * if path is added through uev_add_path, pgindex can be unset.
 974          * next update_strings() will set it, upon map reload event.
 975          *
 976          * we can safely return here, because upon map reload, all
 977          * PG will be enabled.
 978          */
 979         if (!pp->mpp->pg || !pp->pgindex)
 980                 return;
 981
 982         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
 983
 984         if (pgp->status == PGSTATE_DISABLED) {
 985                 condlog(2, "%s: enable group #%i", pp->mpp->alias, pp->pgindex);
 986                 dm_enablegroup(pp->mpp->alias, pp->pgindex);
 987         }
 988 }
 989
 990 static void
 991 mpvec_garbage_collector (struct vectors * vecs)
 992 {
 993         struct multipath * mpp;
 994         unsigned int i;
 995
 996         if (!vecs->mpvec)
 997                 return;
 998
 999         vector_foreach_slot (vecs->mpvec, mpp, i) {
1000                 if (mpp && mpp->alias && !dm_map_present(mpp->alias)) {
1001                         condlog(2, "%s: remove dead map", mpp->alias);
1002                         remove_map_and_stop_waiter(mpp, vecs, 1);
1003                         i--;
1004                 }
1005         }
1006 }
1007
1008 /* This is called after a path has started working again. It the multipath
1009  * device for this path uses the followover failback type, and this is the
1010  * best pathgroup, and this is the first path in the pathgroup to come back
1011  * up, then switch to this pathgroup */
1012 static int
1013 followover_should_failback(struct path * pp)
1014 {
1015         struct pathgroup * pgp;
1016         struct path *pp1;
1017         int i;
1018
1019         if (pp->mpp->pgfailback != -FAILBACK_FOLLOWOVER ||
1020             !pp->mpp->pg || !pp->pgindex ||
1021             pp->pgindex != pp->mpp->bestpg)
1022                 return 0;
1023
1024         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
1025         vector_foreach_slot(pgp->paths, pp1, i) {
1026                 if (pp1 == pp)
1027                         continue;
1028                 if (pp1->chkrstate != PATH_DOWN && pp1->chkrstate != PATH_SHAKY)
1029                         return 0;
1030         }
1031         return 1;
1032 }
1033
1034 static void
1035 defered_failback_tick (vector mpvec)
1036 {
1037         struct multipath * mpp;
1038         unsigned int i;
1039
1040         vector_foreach_slot (mpvec, mpp, i) {
1041                 /*
1042                  * defered failback getting sooner
1043                  */
1044                 if (mpp->pgfailback > 0 && mpp->failback_tick > 0) {
1045                         mpp->failback_tick--;
1046
1047                         if (!mpp->failback_tick && need_switch_pathgroup(mpp, 1))
1048                                 switch_pathgroup(mpp);
1049                 }
1050         }
1051 }
1052
1053 static void
1054 retry_count_tick(vector mpvec)
1055 {
1056         struct multipath *mpp;
1057         unsigned int i;
1058
1059         vector_foreach_slot (mpvec, mpp, i) {
1060                 if (mpp->retry_tick) {
1061                         mpp->stat_total_queueing_time++;
1062                         condlog(4, "%s: Retrying.. No active path", mpp->alias);
1063                         if(--mpp->retry_tick == 0) {
1064                                 dm_queue_if_no_path(mpp->alias, 0);
1065                                 condlog(2, "%s: Disable queueing", mpp->alias);
1066                         }
1067                 }
1068         }
1069 }
1070
1071 int update_prio(struct path *pp, int refresh_all)
1072 {
1073         int oldpriority;
1074         struct path *pp1;
1075         struct pathgroup * pgp;
1076         int i, j, changed = 0;
1077
1078         if (refresh_all) {
1079                 vector_foreach_slot (pp->mpp->pg, pgp, i) {
1080                         vector_foreach_slot (pgp->paths, pp1, j) {
1081                                 oldpriority = pp1->priority;
1082                                 pathinfo(pp1, conf->hwtable, DI_PRIO);
1083                                 if (pp1->priority != oldpriority)
1084                                         changed = 1;
1085                         }
1086                 }
1087                 return changed;
1088         }
1089         oldpriority = pp->priority;
1090         pathinfo(pp, conf->hwtable, DI_PRIO);
1091
1092         if (pp->priority == oldpriority)
1093                 return 0;
1094         return 1;
1095 }
1096
1097 int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
1098 {
1099         if (reload_map(vecs, mpp, refresh))
1100                 return 1;
1101
1102         dm_lib_release();
1103         if (setup_multipath(vecs, mpp) != 0)
1104                 return 1;
1105         sync_map_state(mpp);
1106
1107         return 0;
1108 }
1109
1110 /*
1111  * Returns '1' if the path has been checked, '0' otherwise
1112  */
1113 int
1114 check_path (struct vectors * vecs, struct path * pp)
1115 {
1116         int newstate;
1117         int new_path_up = 0;
1118         int chkr_new_path_up = 0;
1119         int oldchkrstate = pp->chkrstate;
1120
1121         if (!pp->mpp)
1122                 return 0;
1123
1124         if (pp->tick && --pp->tick)
1125                 return 0; /* don't check this path yet */
1126
1127         /*
1128          * provision a next check soonest,
1129          * in case we exit abnormaly from here
1130          */
1131         pp->tick = conf->checkint;
1132
1133         newstate = path_offline(pp);
1134         if (newstate == PATH_REMOVED) {
1135                 condlog(2, "%s: remove path (checker)", pp->dev);
1136                 ev_remove_path(pp, vecs);
1137                 return 0;
1138         }
1139         if (newstate == PATH_UP)
1140                 newstate = get_state(pp, 1);
1141         else
1142                 checker_clear_message(&pp->checker);
1143
1144         if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
1145                 condlog(2, "%s: unusable path", pp->dev);
1146                 pathinfo(pp, conf->hwtable, 0);
1147                 return 1;
1148         }
1149         if (!pp->mpp) {
1150                 if (!strlen(pp->wwid) &&
1151                     (newstate == PATH_UP || newstate == PATH_GHOST)) {
1152                         condlog(2, "%s: add missing path", pp->dev);
1153                         if (pathinfo(pp, conf->hwtable, DI_ALL) == 0) {
1154                                 ev_add_path(pp, vecs);
1155                                 pp->tick = 1;
1156                         }
1157                 }
1158                 return 0;
1159         }
1160         /*
1161          * Async IO in flight. Keep the previous path state
1162          * and reschedule as soon as possible
1163          */
1164         if (newstate == PATH_PENDING) {
1165                 pp->tick = 1;
1166                 return 0;
1167         }
1168         /*
1169          * Synchronize with kernel state
1170          */
1171         if (update_multipath_strings(pp->mpp, vecs->pathvec)) {
1172                 condlog(1, "%s: Could not synchronize with kernel state",
1173                         pp->dev);
1174                 pp->dmstate = PSTATE_UNDEF;
1175         }
1176         pp->chkrstate = newstate;
1177         if (newstate != pp->state) {
1178                 int oldstate = pp->state;
1179                 pp->state = newstate;
1180
1181                 if (strlen(checker_message(&pp->checker)))
1182                         LOG_MSG(1, checker_message(&pp->checker));
1183
1184                 /*
1185                  * upon state change, reset the checkint
1186                  * to the shortest delay
1187                  */
1188                 pp->checkint = conf->checkint;
1189
1190                 if (newstate == PATH_DOWN || newstate == PATH_SHAKY) {
1191                         /*
1192                          * proactively fail path in the DM
1193                          */
1194                         if (oldstate == PATH_UP ||
1195                             oldstate == PATH_GHOST)
1196                                 fail_path(pp, 1);
1197                         else
1198                                 fail_path(pp, 0);
1199
1200                         /*
1201                          * cancel scheduled failback
1202                          */
1203                         pp->mpp->failback_tick = 0;
1204
1205                         pp->mpp->stat_path_failures++;
1206                         return 1;
1207                 }
1208
1209                 if(newstate == PATH_UP || newstate == PATH_GHOST){
1210                         if ( pp->mpp && pp->mpp->prflag ){
1211                                 /*
1212                                  * Check Persistent Reservation.
1213                                  */
1214                         condlog(2, "%s: checking persistent reservation "
1215                                 "registration", pp->dev);
1216                         mpath_pr_event_handle(pp);
1217                         }
1218                 }
1219
1220                 /*
1221                  * reinstate this path
1222                  */
1223                 if (oldstate != PATH_UP &&
1224                     oldstate != PATH_GHOST)
1225                         reinstate_path(pp, 1);
1226                 else
1227                         reinstate_path(pp, 0);
1228
1229                 new_path_up = 1;
1230
1231                 if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST)
1232                         chkr_new_path_up = 1;
1233
1234                 /*
1235                  * if at least one path is up in a group, and
1236                  * the group is disabled, re-enable it
1237                  */
1238                 if (newstate == PATH_UP)
1239                         enable_group(pp);
1240         }
1241         else if (newstate == PATH_UP || newstate == PATH_GHOST) {
1242                 if (pp->dmstate == PSTATE_FAILED ||
1243                     pp->dmstate == PSTATE_UNDEF) {
1244                         /* Clear IO errors */
1245                         reinstate_path(pp, 0);
1246                 } else {
1247                         LOG_MSG(4, checker_message(&pp->checker));
1248                         if (pp->checkint != conf->max_checkint) {
1249                                 /*
1250                                  * double the next check delay.
1251                                  * max at conf->max_checkint
1252                                  */
1253                                 if (pp->checkint < (conf->max_checkint / 2))
1254                                         pp->checkint = 2 * pp->checkint;
1255                                 else
1256                                         pp->checkint = conf->max_checkint;
1257
1258                                 condlog(4, "%s: delay next check %is",
1259                                         pp->dev_t, pp->checkint);
1260                         }
1261                         pp->tick = pp->checkint;
1262                 }
1263         }
1264         else if (newstate == PATH_DOWN &&
1265                  strlen(checker_message(&pp->checker))) {
1266                 if (conf->log_checker_err == LOG_CHKR_ERR_ONCE)
1267                         LOG_MSG(3, checker_message(&pp->checker));
1268                 else
1269                         LOG_MSG(2, checker_message(&pp->checker));
1270         }
1271
1272         pp->state = newstate;
1273
1274         /*
1275          * path prio refreshing
1276          */
1277         condlog(4, "path prio refresh");
1278
1279         if (update_prio(pp, new_path_up) &&
1280             (pp->mpp->pgpolicyfn == (pgpolicyfn *)group_by_prio) &&
1281              pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
1282                 update_path_groups(pp->mpp, vecs, !new_path_up);
1283         else if (need_switch_pathgroup(pp->mpp, 0)) {
1284                 if (pp->mpp->pgfailback > 0 &&
1285                     (new_path_up || pp->mpp->failback_tick <= 0))
1286                         pp->mpp->failback_tick =
1287                                 pp->mpp->pgfailback + 1;
1288                 else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE ||
1289                          (chkr_new_path_up && followover_should_failback(pp)))
1290                         switch_pathgroup(pp->mpp);
1291         }
1292         return 1;
1293 }
1294
1295 static void *
1296 checkerloop (void *ap)
1297 {
1298         struct vectors *vecs;
1299         struct path *pp;
1300         int count = 0;
1301         unsigned int i;
1302
1303         mlockall(MCL_CURRENT | MCL_FUTURE);
1304         vecs = (struct vectors *)ap;
1305         condlog(2, "path checkers start up");
1306
1307         /*
1308          * init the path check interval
1309          */
1310         vector_foreach_slot (vecs->pathvec, pp, i) {
1311                 pp->checkint = conf->checkint;
1312         }
1313
1314         while (1) {
1315                 struct timeval diff_time, start_time, end_time;
1316                 int num_paths = 0;
1317
1318                 if (gettimeofday(&start_time, NULL) != 0)
1319                         start_time.tv_sec = 0;
1320                 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1321                 lock(vecs->lock);
1322                 pthread_testcancel();
1323                 condlog(4, "tick");
1324 #ifdef USE_SYSTEMD
1325                 if (conf->watchdog)
1326                         sd_notify(0, "WATCHDOG=1");
1327 #endif
1328                 if (vecs->pathvec) {
1329                         vector_foreach_slot (vecs->pathvec, pp, i) {
1330                                 num_paths += check_path(vecs, pp);
1331                         }
1332                 }
1333                 if (vecs->mpvec) {
1334                         defered_failback_tick(vecs->mpvec);
1335                         retry_count_tick(vecs->mpvec);
1336                 }
1337                 if (count)
1338                         count--;
1339                 else {
1340                         condlog(4, "map garbage collection");
1341                         mpvec_garbage_collector(vecs);
1342                         count = MAPGCINT;
1343                 }
1344
1345                 lock_cleanup_pop(vecs->lock);
1346                 if (start_time.tv_sec &&
1347                     gettimeofday(&end_time, NULL) == 0 &&
1348                     num_paths) {
1349                         timersub(&end_time, &start_time, &diff_time);
1350                         condlog(3, "checked %d path%s in %lu.%06lu secs",
1351                                 num_paths, num_paths > 1 ? "s" : "",
1352                                 diff_time.tv_sec, diff_time.tv_usec);
1353                 }
1354                 sleep(1);
1355         }
1356         return NULL;
1357 }
1358
1359 int
1360 configure (struct vectors * vecs, int start_waiters)
1361 {
1362         struct multipath * mpp;
1363         struct path * pp;
1364         vector mpvec;
1365         int i;
1366
1367         if (!vecs->pathvec && !(vecs->pathvec = vector_alloc()))
1368                 return 1;
1369
1370         if (!vecs->mpvec && !(vecs->mpvec = vector_alloc()))
1371                 return 1;
1372
1373         if (!(mpvec = vector_alloc()))
1374                 return 1;
1375
1376         /*
1377          * probe for current path (from sysfs) and map (from dm) sets
1378          */
1379         path_discovery(vecs->pathvec, conf, DI_ALL);
1380
1381         vector_foreach_slot (vecs->pathvec, pp, i){
1382                 if (filter_path(conf, pp) > 0){
1383                         vector_del_slot(vecs->pathvec, i);
1384                         free_path(pp);
1385                         i--;
1386                 }
1387                 else
1388                         pp->checkint = conf->checkint;
1389         }
1390         if (map_discovery(vecs))
1391                 return 1;
1392
1393         /*
1394          * create new set of maps & push changed ones into dm
1395          */
1396         if (coalesce_paths(vecs, mpvec, NULL, 1))
1397                 return 1;
1398
1399         /*
1400          * may need to remove some maps which are no longer relevant
1401          * e.g., due to blacklist changes in conf file
1402          */
1403         if (coalesce_maps(vecs, mpvec))
1404                 return 1;
1405
1406         dm_lib_release();
1407
1408         sync_maps_state(mpvec);
1409         vector_foreach_slot(mpvec, mpp, i){
1410                 remember_wwid(mpp->wwid);
1411                 update_map_pr(mpp);
1412         }
1413
1414         /*
1415          * purge dm of old maps
1416          */
1417         remove_maps(vecs);
1418
1419         /*
1420          * save new set of maps formed by considering current path state
1421          */
1422         vector_free(vecs->mpvec);
1423         vecs->mpvec = mpvec;
1424
1425         /*
1426          * start dm event waiter threads for these new maps
1427          */
1428         vector_foreach_slot(vecs->mpvec, mpp, i) {
1429                 if (setup_multipath(vecs, mpp))
1430                         return 1;
1431                 if (start_waiters)
1432                         if (start_waiter_thread(mpp, vecs))
1433                                 return 1;
1434         }
1435         return 0;
1436 }
1437
1438 int
1439 reconfigure (struct vectors * vecs)
1440 {
1441         struct config * old = conf;
1442         int retval = 1;
1443
1444         /*
1445          * free old map and path vectors ... they use old conf state
1446          */
1447         if (VECTOR_SIZE(vecs->mpvec))
1448                 remove_maps_and_stop_waiters(vecs);
1449
1450         if (VECTOR_SIZE(vecs->pathvec))
1451                 free_pathvec(vecs->pathvec, FREE_PATHS);
1452
1453         vecs->pathvec = NULL;
1454         conf = NULL;
1455
1456         /* Re-read any timezone changes */
1457         tzset();
1458
1459         if (!load_config(DEFAULT_CONFIGFILE, udev)) {
1460                 dm_drv_version(conf->version, TGT_MPATH);
1461                 conf->verbosity = old->verbosity;
1462                 conf->daemon = 1;
1463                 configure(vecs, 1);
1464                 free_config(old);
1465                 retval = 0;
1466         }
1467
1468         return retval;
1469 }
1470
1471 static struct vectors *
1472 init_vecs (void)
1473 {
1474         struct vectors * vecs;
1475
1476         vecs = (struct vectors *)MALLOC(sizeof(struct vectors));
1477
1478         if (!vecs)
1479                 return NULL;
1480
1481         vecs->lock.mutex =
1482                 (pthread_mutex_t *)MALLOC(sizeof(pthread_mutex_t));
1483
1484         if (!vecs->lock.mutex)
1485                 goto out;
1486
1487         pthread_mutex_init(vecs->lock.mutex, NULL);
1488         vecs->lock.depth = 0;
1489
1490         return vecs;
1491
1492 out:
1493         FREE(vecs);
1494         condlog(0, "failed to init paths");
1495         return NULL;
1496 }
1497
1498 static void *
1499 signal_set(int signo, void (*func) (int))
1500 {
1501         int r;
1502         struct sigaction sig;
1503         struct sigaction osig;
1504
1505         sig.sa_handler = func;
1506         sigemptyset(&sig.sa_mask);
1507         sig.sa_flags = 0;
1508
1509         r = sigaction(signo, &sig, &osig);
1510
1511         if (r < 0)
1512                 return (SIG_ERR);
1513         else
1514                 return (osig.sa_handler);
1515 }
1516
1517 void
1518 handle_signals(void)
1519 {
1520         if (reconfig_sig && running_state == DAEMON_RUNNING) {
1521                 condlog(2, "reconfigure (signal)");
1522                 pthread_cleanup_push(cleanup_lock,
1523                                 &gvecs->lock);
1524                 lock(gvecs->lock);
1525                 pthread_testcancel();
1526                 reconfigure(gvecs);
1527                 lock_cleanup_pop(gvecs->lock);
1528         }
1529         if (log_reset_sig) {
1530                 condlog(2, "reset log (signal)");
1531                 pthread_mutex_lock(&logq_lock);
1532                 log_reset("multipathd");
1533                 pthread_mutex_unlock(&logq_lock);
1534         }
1535         reconfig_sig = 0;
1536         log_reset_sig = 0;
1537 }
1538
1539 static void
1540 sighup (int sig)
1541 {
1542         reconfig_sig = 1;
1543 }
1544
1545 static void
1546 sigend (int sig)
1547 {
1548         exit_daemon();
1549 }
1550
1551 static void
1552 sigusr1 (int sig)
1553 {
1554         log_reset_sig = 1;
1555 }
1556
1557 static void
1558 sigusr2 (int sig)
1559 {
1560         condlog(3, "SIGUSR2 received");
1561 }
1562
1563 static void
1564 signal_init(void)
1565 {
1566         sigset_t set;
1567
1568         sigemptyset(&set);
1569         sigaddset(&set, SIGHUP);
1570         sigaddset(&set, SIGUSR1);
1571         sigaddset(&set, SIGUSR2);
1572         pthread_sigmask(SIG_BLOCK, &set, NULL);
1573
1574         signal_set(SIGHUP, sighup);
1575         signal_set(SIGUSR1, sigusr1);
1576         signal_set(SIGUSR2, sigusr2);
1577         signal_set(SIGINT, sigend);
1578         signal_set(SIGTERM, sigend);
1579         signal(SIGPIPE, SIG_IGN);
1580 }
1581
1582 static void
1583 setscheduler (void)
1584 {
1585         int res;
1586         static struct sched_param sched_param = {
1587                 .sched_priority = 99
1588         };
1589
1590         res = sched_setscheduler (0, SCHED_RR, &sched_param);
1591
1592         if (res == -1)
1593                 condlog(LOG_WARNING, "Could not set SCHED_RR at priority 99");
1594         return;
1595 }
1596
1597 static void
1598 set_oom_adj (void)
1599 {
1600 #ifdef OOM_SCORE_ADJ_MIN
1601         int retry = 1;
1602         char *file = "/proc/self/oom_score_adj";
1603         int score = OOM_SCORE_ADJ_MIN;
1604 #else
1605         int retry = 0;
1606         char *file = "/proc/self/oom_adj";
1607         int score = OOM_ADJUST_MIN;
1608 #endif
1609         FILE *fp;
1610         struct stat st;
1611         char *envp;
1612
1613         envp = getenv("OOMScoreAdjust");
1614         if (envp) {
1615                 condlog(3, "Using systemd provided OOMScoreAdjust");
1616                 return;
1617         }
1618         do {
1619                 if (stat(file, &st) == 0){
1620                         fp = fopen(file, "w");
1621                         if (!fp) {
1622                                 condlog(0, "couldn't fopen %s : %s", file,
1623                                         strerror(errno));
1624                                 return;
1625                         }
1626                         fprintf(fp, "%i", score);
1627                         fclose(fp);
1628                         return;
1629                 }
1630                 if (errno != ENOENT) {
1631                         condlog(0, "couldn't stat %s : %s", file,
1632                                 strerror(errno));
1633                         return;
1634                 }
1635 #ifdef OOM_ADJUST_MIN
1636                 file = "/proc/self/oom_adj";
1637                 score = OOM_ADJUST_MIN;
1638 #else
1639                 retry = 0;
1640 #endif
1641         } while (retry--);
1642         condlog(0, "couldn't adjust oom score");
1643 }
1644
1645 static int
1646 child (void * param)
1647 {
1648         pthread_t check_thr, uevent_thr, uxlsnr_thr, uevq_thr;
1649         pthread_attr_t log_attr, misc_attr, uevent_attr;
1650         struct vectors * vecs;
1651         struct multipath * mpp;
1652         int i;
1653 #ifdef USE_SYSTEMD
1654         unsigned long checkint;
1655 #endif
1656         int rc, pid_rc;
1657         char *envp;
1658
1659         mlockall(MCL_CURRENT | MCL_FUTURE);
1660         sem_init(&exit_sem, 0, 0);
1661         signal_init();
1662
1663         udev = udev_new();
1664
1665         setup_thread_attr(&misc_attr, 64 * 1024, 1);
1666         setup_thread_attr(&uevent_attr, 128 * 1024, 1);
1667         setup_thread_attr(&waiter_attr, 32 * 1024, 1);
1668
1669         if (logsink == 1) {
1670                 setup_thread_attr(&log_attr, 64 * 1024, 0);
1671                 log_thread_start(&log_attr);
1672                 pthread_attr_destroy(&log_attr);
1673         }
1674
1675         running_state = DAEMON_START;
1676
1677 #ifdef USE_SYSTEMD
1678         sd_notify(0, "STATUS=startup");
1679 #endif
1680         condlog(2, "--------start up--------");
1681         condlog(2, "read " DEFAULT_CONFIGFILE);
1682
1683         if (load_config(DEFAULT_CONFIGFILE, udev))
1684                 goto failed;
1685
1686         dm_drv_version(conf->version, TGT_MPATH);
1687         if (init_checkers()) {
1688                 condlog(0, "failed to initialize checkers");
1689                 goto failed;
1690         }
1691         if (init_prio()) {
1692                 condlog(0, "failed to initialize prioritizers");
1693                 goto failed;
1694         }
1695
1696         setlogmask(LOG_UPTO(conf->verbosity + 3));
1697
1698         envp = getenv("LimitNOFILE");
1699
1700         if (envp) {
1701                 condlog(2,"Using systemd provided open fds limit of %s", envp);
1702         } else if (conf->max_fds) {
1703                 struct rlimit fd_limit;
1704
1705                 if (getrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
1706                         condlog(0, "can't get open fds limit: %s",
1707                                 strerror(errno));
1708                         fd_limit.rlim_cur = 0;
1709                         fd_limit.rlim_max = 0;
1710                 }
1711                 if (fd_limit.rlim_cur < conf->max_fds) {
1712                         fd_limit.rlim_cur = conf->max_fds;
1713                         if (fd_limit.rlim_max < conf->max_fds)
1714                                 fd_limit.rlim_max = conf->max_fds;
1715                         if (setrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
1716                                 condlog(0, "can't set open fds limit to "
1717                                         "%lu/%lu : %s",
1718                                         fd_limit.rlim_cur, fd_limit.rlim_max,
1719                                         strerror(errno));
1720                         } else {
1721                                 condlog(3, "set open fds limit to %lu/%lu",
1722                                         fd_limit.rlim_cur, fd_limit.rlim_max);
1723                         }
1724                 }
1725
1726         }
1727
1728         vecs = gvecs = init_vecs();
1729         if (!vecs)
1730                 goto failed;
1731
1732         setscheduler();
1733         set_oom_adj();
1734
1735         conf->daemon = 1;
1736         udev_set_sync_support(0);
1737 #ifdef USE_SYSTEMD
1738         envp = getenv("WATCHDOG_USEC");
1739         if (envp && sscanf(envp, "%lu", &checkint) == 1) {
1740                 /* Value is in microseconds */
1741                 conf->max_checkint = checkint / 1000000;
1742                 /* Rescale checkint */
1743                 if (conf->checkint > conf->max_checkint)
1744                         conf->checkint = conf->max_checkint;
1745                 else
1746                         conf->checkint = conf->max_checkint / 4;
1747                 condlog(3, "enabling watchdog, interval %d max %d",
1748                         conf->checkint, conf->max_checkint);
1749                 conf->watchdog = conf->checkint;
1750         }
1751 #endif
1752         /*
1753          * Start uevent listener early to catch events
1754          */
1755         if ((rc = pthread_create(&uevent_thr, &uevent_attr, ueventloop, udev))) {
1756                 condlog(0, "failed to create uevent thread: %d", rc);
1757                 goto failed;
1758         }
1759         pthread_attr_destroy(&uevent_attr);
1760         if ((rc = pthread_create(&uxlsnr_thr, &misc_attr, uxlsnrloop, vecs))) {
1761                 condlog(0, "failed to create cli listener: %d", rc);
1762                 goto failed;
1763         }
1764         /*
1765          * fetch and configure both paths and multipaths
1766          */
1767 #ifdef USE_SYSTEMD
1768         sd_notify(0, "STATUS=configure");
1769 #endif
1770         running_state = DAEMON_CONFIGURE;
1771
1772         lock(vecs->lock);
1773         if (configure(vecs, 1)) {
1774                 unlock(vecs->lock);
1775                 condlog(0, "failure during configuration");
1776                 goto failed;
1777         }
1778         unlock(vecs->lock);
1779
1780         /*
1781          * start threads
1782          */
1783         if ((rc = pthread_create(&check_thr, &misc_attr, checkerloop, vecs))) {
1784                 condlog(0,"failed to create checker loop thread: %d", rc);
1785                 goto failed;
1786         }
1787         if ((rc = pthread_create(&uevq_thr, &misc_attr, uevqloop, vecs))) {
1788                 condlog(0, "failed to create uevent dispatcher: %d", rc);
1789                 goto failed;
1790         }
1791         pthread_attr_destroy(&misc_attr);
1792
1793         /* Startup complete, create logfile */
1794         pid_rc = pidfile_create(DEFAULT_PIDFILE, daemon_pid);
1795         /* Ignore errors, we can live without */
1796
1797         running_state = DAEMON_RUNNING;
1798 #ifdef USE_SYSTEMD
1799         sd_notify(0, "READY=1\nSTATUS=running");
1800 #endif
1801
1802         /*
1803          * exit path
1804          */
1805         while(sem_wait(&exit_sem) != 0); /* Do nothing */
1806
1807 #ifdef USE_SYSTEMD
1808         sd_notify(0, "STATUS=shutdown");
1809 #endif
1810         running_state = DAEMON_SHUTDOWN;
1811         lock(vecs->lock);
1812         if (conf->queue_without_daemon == QUE_NO_DAEMON_OFF)
1813                 vector_foreach_slot(vecs->mpvec, mpp, i)
1814                         dm_queue_if_no_path(mpp->alias, 0);
1815         remove_maps_and_stop_waiters(vecs);
1816         unlock(vecs->lock);
1817
1818         pthread_cancel(check_thr);
1819         pthread_cancel(uevent_thr);
1820         pthread_cancel(uxlsnr_thr);
1821         pthread_cancel(uevq_thr);
1822
1823         lock(vecs->lock);
1824         free_pathvec(vecs->pathvec, FREE_PATHS);
1825         vecs->pathvec = NULL;
1826         unlock(vecs->lock);
1827         /* Now all the waitevent threads will start rushing in. */
1828         while (vecs->lock.depth > 0) {
1829                 sleep (1); /* This is weak. */
1830                 condlog(3, "Have %d wait event checkers threads to de-alloc,"
1831                         " waiting...", vecs->lock.depth);
1832         }
1833         pthread_mutex_destroy(vecs->lock.mutex);
1834         FREE(vecs->lock.mutex);
1835         vecs->lock.depth = 0;
1836         vecs->lock.mutex = NULL;
1837         FREE(vecs);
1838         vecs = NULL;
1839
1840         cleanup_checkers();
1841         cleanup_prio();
1842
1843         dm_lib_release();
1844         dm_lib_exit();
1845
1846         /* We're done here */
1847         if (!pid_rc) {
1848                 condlog(3, "unlink pidfile");
1849                 unlink(DEFAULT_PIDFILE);
1850         }
1851
1852         condlog(2, "--------shut down-------");
1853
1854         if (logsink == 1)
1855                 log_thread_stop();
1856
1857         /*
1858          * Freeing config must be done after condlog() and dm_lib_exit(),
1859          * because logging functions like dlog() and dm_write_log()
1860          * reference the config.
1861          */
1862         free_config(conf);
1863         conf = NULL;
1864         udev_unref(udev);
1865         udev = NULL;
1866 #ifdef _DEBUG_
1867         dbg_free_final(NULL);
1868 #endif
1869
1870 #ifdef USE_SYSTEMD
1871         sd_notify(0, "ERRNO=0");
1872 #endif
1873         exit(0);
1874
1875 failed:
1876 #ifdef USE_SYSTEMD
1877         sd_notify(0, "ERRNO=1");
1878 #endif
1879         exit(1);
1880 }
1881
1882 static int
1883 daemonize(void)
1884 {
1885         int pid;
1886         int dev_null_fd;
1887
1888         if( (pid = fork()) < 0){
1889                 fprintf(stderr, "Failed first fork : %s\n", strerror(errno));
1890                 return -1;
1891         }
1892         else if (pid != 0)
1893                 return pid;
1894
1895         setsid();
1896
1897         if ( (pid = fork()) < 0)
1898                 fprintf(stderr, "Failed second fork : %s\n", strerror(errno));
1899         else if (pid != 0)
1900                 _exit(0);
1901
1902         if (chdir("/") < 0)
1903                 fprintf(stderr, "cannot chdir to '/', continuing\n");
1904
1905         dev_null_fd = open("/dev/null", O_RDWR);
1906         if (dev_null_fd < 0){
1907                 fprintf(stderr, "cannot open /dev/null for input & output : %s\n",
1908                         strerror(errno));
1909                 _exit(0);
1910         }
1911
1912         close(STDIN_FILENO);
1913         if (dup(dev_null_fd) < 0) {
1914                 fprintf(stderr, "cannot dup /dev/null to stdin : %s\n",
1915                         strerror(errno));
1916                 _exit(0);
1917         }
1918         close(STDOUT_FILENO);
1919         if (dup(dev_null_fd) < 0) {
1920                 fprintf(stderr, "cannot dup /dev/null to stdout : %s\n",
1921                         strerror(errno));
1922                 _exit(0);
1923         }
1924         close(STDERR_FILENO);
1925         if (dup(dev_null_fd) < 0) {
1926                 fprintf(stderr, "cannot dup /dev/null to stderr : %s\n",
1927                         strerror(errno));
1928                 _exit(0);
1929         }
1930         close(dev_null_fd);
1931         daemon_pid = getpid();
1932         return 0;
1933 }
1934
1935 int
1936 main (int argc, char *argv[])
1937 {
1938         extern char *optarg;
1939         extern int optind;
1940         int arg;
1941         int err;
1942
1943         logsink = 1;
1944         running_state = DAEMON_INIT;
1945         dm_init();
1946
1947         if (getuid() != 0) {
1948                 fprintf(stderr, "need to be root\n");
1949                 exit(1);
1950         }
1951
1952         /* make sure we don't lock any path */
1953         if (chdir("/") < 0)
1954                 fprintf(stderr, "can't chdir to root directory : %s\n",
1955                         strerror(errno));
1956         umask(umask(077) | 022);
1957
1958         conf = alloc_config();
1959
1960         if (!conf)
1961                 exit(1);
1962
1963         while ((arg = getopt(argc, argv, ":dsv:k::")) != EOF ) {
1964         switch(arg) {
1965                 case 'd':
1966                         logsink = 0;
1967                         //debug=1; /* ### comment me out ### */
1968                         break;
1969                 case 'v':
1970                         if (sizeof(optarg) > sizeof(char *) ||
1971                             !isdigit(optarg[0]))
1972                                 exit(1);
1973
1974                         conf->verbosity = atoi(optarg);
1975                         break;
1976                 case 's':
1977                         logsink = -1;
1978                         break;
1979                 case 'k':
1980                         uxclnt(optarg);
1981                         exit(0);
1982                 default:
1983                         ;
1984                 }
1985         }
1986         if (optind < argc) {
1987                 char cmd[CMDSIZE];
1988                 char * s = cmd;
1989                 char * c = s;
1990
1991                 while (optind < argc) {
1992                         if (strchr(argv[optind], ' '))
1993                                 c += snprintf(c, s + CMDSIZE - c, "\"%s\" ", argv[optind]);
1994                         else
1995                                 c += snprintf(c, s + CMDSIZE - c, "%s ", argv[optind]);
1996                         optind++;
1997                 }
1998                 c += snprintf(c, s + CMDSIZE - c, "\n");
1999                 uxclnt(s);
2000                 exit(0);
2001         }
2002
2003         if (logsink < 1)
2004                 err = 0;
2005         else
2006                 err = daemonize();
2007
2008         if (err < 0)
2009                 /* error */
2010                 exit(1);
2011         else if (err > 0)
2012                 /* parent dies */
2013                 exit(0);
2014         else
2015                 /* child lives */
2016                 return (child(NULL));
2017 }
2018
2019 void *  mpath_pr_event_handler_fn (void * pathp )
2020 {
2021         struct multipath * mpp;
2022         int i,j, ret, isFound;
2023         struct path * pp = (struct path *)pathp;
2024         unsigned char *keyp;
2025         uint64_t prkey;
2026         struct prout_param_descriptor *param;
2027         struct prin_resp *resp;
2028
2029         mpp = pp->mpp;
2030
2031         resp = mpath_alloc_prin_response(MPATH_PRIN_RKEY_SA);
2032         if (!resp){
2033                 condlog(0,"%s Alloc failed for prin response", pp->dev);
2034                 return NULL;
2035         }
2036
2037         ret = prin_do_scsi_ioctl(pp->dev, MPATH_PRIN_RKEY_SA, resp, 0);
2038         if (ret != MPATH_PR_SUCCESS )
2039         {
2040                 condlog(0,"%s : pr in read keys service action failed. Error=%d", pp->dev, ret);
2041                 goto out;
2042         }
2043
2044         condlog(3, " event pr=%d addlen=%d",resp->prin_descriptor.prin_readkeys.prgeneration,
2045                         resp->prin_descriptor.prin_readkeys.additional_length );
2046
2047         if (resp->prin_descriptor.prin_readkeys.additional_length == 0 )
2048         {
2049                 condlog(1, "%s: No key found. Device may not be registered.", pp->dev);
2050                 ret = MPATH_PR_SUCCESS;
2051                 goto out;
2052         }
2053         prkey = 0;
2054         keyp = (unsigned char *)mpp->reservation_key;
2055         for (j = 0; j < 8; ++j) {
2056                 if (j > 0)
2057                         prkey <<= 8;
2058                 prkey |= *keyp;
2059                 ++keyp;
2060         }
2061         condlog(2, "Multipath  reservation_key: 0x%" PRIx64 " ", prkey);
2062
2063         isFound =0;
2064         for (i = 0; i < resp->prin_descriptor.prin_readkeys.additional_length/8; i++ )
2065         {
2066                 condlog(2, "PR IN READKEYS[%d]  reservation key:",i);
2067                 dumpHex((char *)&resp->prin_descriptor.prin_readkeys.key_list[i*8], 8 , -1);
2068                 if (!memcmp(mpp->reservation_key, &resp->prin_descriptor.prin_readkeys.key_list[i*8], 8))
2069                 {
2070                         condlog(2, "%s: pr key found in prin readkeys response", mpp->alias);
2071                         isFound =1;
2072                         break;
2073                 }
2074         }
2075         if (!isFound)
2076         {
2077                 condlog(0, "%s: Either device not registered or ", pp->dev);
2078                 condlog(0, "host is not authorised for registration. Skip path");
2079                 ret = MPATH_PR_OTHER;
2080                 goto out;
2081         }
2082
2083         param= malloc(sizeof(struct prout_param_descriptor));
2084         memset(param, 0 , sizeof(struct prout_param_descriptor));
2085
2086         for (j = 7; j >= 0; --j) {
2087                 param->sa_key[j] = (prkey & 0xff);
2088                 prkey >>= 8;
2089         }
2090         param->num_transportid = 0;
2091
2092         condlog(3, "device %s:%s", pp->dev, pp->mpp->wwid);
2093
2094         ret = prout_do_scsi_ioctl(pp->dev, MPATH_PROUT_REG_IGN_SA, 0, 0, param, 0);
2095         if (ret != MPATH_PR_SUCCESS )
2096         {
2097                 condlog(0,"%s: Reservation registration failed. Error: %d", pp->dev, ret);
2098         }
2099         mpp->prflag = 1;
2100
2101         free(param);
2102 out:
2103         free(resp);
2104         return NULL;
2105 }
2106
2107 int mpath_pr_event_handle(struct path *pp)
2108 {
2109         pthread_t thread;
2110         int rc;
2111         pthread_attr_t attr;
2112         struct multipath * mpp;
2113
2114         mpp = pp->mpp;
2115
2116         if (!mpp->reservation_key)
2117                 return -1;
2118
2119         pthread_attr_init(&attr);
2120         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
2121
2122         rc = pthread_create(&thread, NULL , mpath_pr_event_handler_fn, pp);
2123         if (rc) {
2124                 condlog(0, "%s: ERROR; return code from pthread_create() is %d", pp->dev, rc);
2125                 return -1;
2126         }
2127         pthread_attr_destroy(&attr);
2128         rc = pthread_join(thread, NULL);
2129         return 0;
2130 }
2131