67459769c146825cac56515ece8723c66844b409
[platform/upstream/multipath-tools.git] / libmultipath / configure.c
1 /*
2  * Copyright (c) 2003, 2004, 2005 Christophe Varoqui
3  * Copyright (c) 2005 Benjamin Marzinski, Redhat
4  * Copyright (c) 2005 Kiyoshi Ueda, NEC
5  * Copyright (c) 2005 Patrick Caulfield, Redhat
6  * Copyright (c) 2005 Edward Goggin, EMC
7  */
8
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <unistd.h>
12 #include <string.h>
13 #include <sys/file.h>
14 #include <errno.h>
15 #include <ctype.h>
16 #include <libdevmapper.h>
17 #include <libudev.h>
18 #include "mpath_cmd.h"
19
20 #include "checkers.h"
21 #include "vector.h"
22 #include "devmapper.h"
23 #include "defaults.h"
24 #include "structs.h"
25 #include "structs_vec.h"
26 #include "dmparser.h"
27 #include "config.h"
28 #include "blacklist.h"
29 #include "propsel.h"
30 #include "discovery.h"
31 #include "debug.h"
32 #include "switchgroup.h"
33 #include "dm-generic.h"
34 #include "print.h"
35 #include "configure.h"
36 #include "pgpolicies.h"
37 #include "dict.h"
38 #include "alias.h"
39 #include "prio.h"
40 #include "util.h"
41 #include "uxsock.h"
42 #include "wwids.h"
43 #include "sysfs.h"
44 #include "io_err_stat.h"
45
46 /* group paths in pg by host adapter
47  */
48 int group_by_host_adapter(struct pathgroup *pgp, vector adapters)
49 {
50         struct adapter_group *agp;
51         struct host_group *hgp;
52         struct path *pp, *pp1;
53         char adapter_name1[SLOT_NAME_SIZE];
54         char adapter_name2[SLOT_NAME_SIZE];
55         int i, j;
56         int found_hostgroup = 0;
57
58         while (VECTOR_SIZE(pgp->paths) > 0) {
59
60                 pp = VECTOR_SLOT(pgp->paths, 0);
61
62                 if (sysfs_get_host_adapter_name(pp, adapter_name1))
63                         goto out;
64                 /* create a new host adapter group
65                  */
66                 agp = alloc_adaptergroup();
67                 if (!agp)
68                         goto out;
69                 agp->pgp = pgp;
70
71                 strlcpy(agp->adapter_name, adapter_name1, SLOT_NAME_SIZE);
72                 store_adaptergroup(adapters, agp);
73
74                 /* create a new host port group
75                  */
76                 hgp = alloc_hostgroup();
77                 if (!hgp)
78                         goto out;
79                 if (store_hostgroup(agp->host_groups, hgp))
80                         goto out;
81
82                 hgp->host_no = pp->sg_id.host_no;
83                 agp->num_hosts++;
84                 if (store_path(hgp->paths, pp))
85                         goto out;
86
87                 hgp->num_paths++;
88                 /* delete path from path group
89                  */
90                 vector_del_slot(pgp->paths, 0);
91
92                 /* add all paths belonging to same host adapter
93                  */
94                 vector_foreach_slot(pgp->paths, pp1, i) {
95                         if (sysfs_get_host_adapter_name(pp1, adapter_name2))
96                                 goto out;
97                         if (strcmp(adapter_name1, adapter_name2) == 0) {
98                                 found_hostgroup = 0;
99                                 vector_foreach_slot(agp->host_groups, hgp, j) {
100                                         if (hgp->host_no == pp1->sg_id.host_no) {
101                                                 if (store_path(hgp->paths, pp1))
102                                                         goto out;
103                                                 hgp->num_paths++;
104                                                 found_hostgroup = 1;
105                                                 break;
106                                         }
107                                 }
108                                 if (!found_hostgroup) {
109                                         /* this path belongs to new host port
110                                          * within this adapter
111                                          */
112                                         hgp = alloc_hostgroup();
113                                         if (!hgp)
114                                                 goto out;
115
116                                         if (store_hostgroup(agp->host_groups, hgp))
117                                                 goto out;
118
119                                         agp->num_hosts++;
120                                         if (store_path(hgp->paths, pp1))
121                                                 goto out;
122
123                                         hgp->host_no = pp1->sg_id.host_no;
124                                         hgp->num_paths++;
125                                 }
126                                 /* delete paths from original path_group
127                                  * as they are added into adapter group now
128                                  */
129                                 vector_del_slot(pgp->paths, i);
130                                 i--;
131                         }
132                 }
133         }
134         return 0;
135
136 out:    /* add back paths into pg as re-ordering failed
137          */
138         vector_foreach_slot(adapters, agp, i) {
139                         vector_foreach_slot(agp->host_groups, hgp, j) {
140                                 while (VECTOR_SIZE(hgp->paths) > 0) {
141                                         pp = VECTOR_SLOT(hgp->paths, 0);
142                                         if (store_path(pgp->paths, pp))
143                                                 condlog(3, "failed to restore "
144                                                 "path %s into path group",
145                                                  pp->dev);
146                                         vector_del_slot(hgp->paths, 0);
147                                 }
148                         }
149                 }
150         free_adaptergroup(adapters);
151         return 1;
152 }
153
154 /* re-order paths in pg by alternating adapters and host ports
155  * for optimized selection
156  */
157 int order_paths_in_pg_by_alt_adapters(struct pathgroup *pgp, vector adapters,
158                  int total_paths)
159 {
160         int next_adapter_index = 0;
161         struct adapter_group *agp;
162         struct host_group *hgp;
163         struct path *pp;
164
165         while (total_paths > 0) {
166                 agp = VECTOR_SLOT(adapters, next_adapter_index);
167                 if (!agp) {
168                         condlog(0, "can't get adapter group %d", next_adapter_index);
169                         return 1;
170                 }
171
172                 hgp = VECTOR_SLOT(agp->host_groups, agp->next_host_index);
173                 if (!hgp) {
174                         condlog(0, "can't get host group %d of adapter group %d", next_adapter_index, agp->next_host_index);
175                         return 1;
176                 }
177
178                 if (!hgp->num_paths) {
179                         agp->next_host_index++;
180                         agp->next_host_index %= agp->num_hosts;
181                         next_adapter_index++;
182                         next_adapter_index %= VECTOR_SIZE(adapters);
183                         continue;
184                 }
185
186                 pp  = VECTOR_SLOT(hgp->paths, 0);
187
188                 if (store_path(pgp->paths, pp))
189                         return 1;
190
191                 total_paths--;
192
193                 vector_del_slot(hgp->paths, 0);
194
195                 hgp->num_paths--;
196
197                 agp->next_host_index++;
198                 agp->next_host_index %= agp->num_hosts;
199                 next_adapter_index++;
200                 next_adapter_index %= VECTOR_SIZE(adapters);
201         }
202
203         /* all paths are added into path_group
204          * in crafted child order
205          */
206         return 0;
207 }
208
209 /* round-robin: order paths in path group to alternate
210  * between all host adapters
211  */
212 int rr_optimize_path_order(struct pathgroup *pgp)
213 {
214         vector adapters;
215         struct path *pp;
216         int total_paths;
217         int i;
218
219         total_paths = VECTOR_SIZE(pgp->paths);
220         vector_foreach_slot(pgp->paths, pp, i) {
221                 if (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP &&
222                         pp->sg_id.proto_id != SCSI_PROTOCOL_SAS &&
223                         pp->sg_id.proto_id != SCSI_PROTOCOL_ISCSI &&
224                         pp->sg_id.proto_id != SCSI_PROTOCOL_SRP) {
225                         /* return success as default path order
226                          * is maintained in path group
227                          */
228                         return 0;
229                 }
230         }
231         adapters = vector_alloc();
232         if (!adapters)
233                 return 0;
234
235         /* group paths in path group by host adapters
236          */
237         if (group_by_host_adapter(pgp, adapters)) {
238                 /* already freed adapters */
239                 condlog(3, "Failed to group paths by adapters");
240                 return 0;
241         }
242
243         /* re-order paths in pg to alternate between adapters and host ports
244          */
245         if (order_paths_in_pg_by_alt_adapters(pgp, adapters, total_paths)) {
246                 condlog(3, "Failed to re-order paths in pg by adapters "
247                         "and host ports");
248                 free_adaptergroup(adapters);
249                 /* return failure as original paths are
250                  * removed form pgp
251                  */
252                 return 1;
253         }
254
255         free_adaptergroup(adapters);
256         return 0;
257 }
258
259 int setup_map(struct multipath *mpp, char **params, struct vectors *vecs)
260 {
261         struct pathgroup * pgp;
262         struct config *conf;
263         int i, marginal_pathgroups;
264         char *save_attr;
265
266         /*
267          * don't bother if devmap size is unknown
268          */
269         if (mpp->size <= 0) {
270                 condlog(3, "%s: devmap size is unknown", mpp->alias);
271                 return 1;
272         }
273
274         if (mpp->disable_queueing && VECTOR_SIZE(mpp->paths) != 0)
275                 mpp->disable_queueing = 0;
276
277         /*
278          * If this map was created with add_map_without_path(),
279          * mpp->hwe might not be set yet.
280          */
281         if (!mpp->hwe)
282                 extract_hwe_from_path(mpp);
283
284         /*
285          * properties selectors
286          *
287          * Ordering matters for some properties:
288          * - features after no_path_retry and retain_hwhandler
289          * - hwhandler after retain_hwhandler
290          * No guarantee that this list is complete, check code in
291          * propsel.c if in doubt.
292          */
293         conf = get_multipath_config();
294         pthread_cleanup_push(put_multipath_config, conf);
295
296         select_pgfailback(conf, mpp);
297         select_pgpolicy(conf, mpp);
298
299         /*
300          * If setup_map() is called from e.g. from reload_map() or resize_map(),
301          * make sure that we don't corrupt attributes.
302          */
303         save_attr = steal_ptr(mpp->selector);
304         select_selector(conf, mpp);
305         if (!mpp->selector)
306                 mpp->selector = save_attr;
307         else
308                 free(save_attr);
309
310         select_no_path_retry(conf, mpp);
311         select_retain_hwhandler(conf, mpp);
312
313         save_attr = steal_ptr(mpp->features);
314         select_features(conf, mpp);
315         if (!mpp->features)
316                 mpp->features = save_attr;
317         else
318                 free(save_attr);
319
320         save_attr = steal_ptr(mpp->hwhandler);
321         select_hwhandler(conf, mpp);
322         if (!mpp->hwhandler)
323                 mpp->hwhandler = save_attr;
324         else
325                 free(save_attr);
326
327         select_rr_weight(conf, mpp);
328         select_minio(conf, mpp);
329         select_mode(conf, mpp);
330         select_uid(conf, mpp);
331         select_gid(conf, mpp);
332         select_fast_io_fail(conf, mpp);
333         select_dev_loss(conf, mpp);
334         select_eh_deadline(conf, mpp);
335         select_reservation_key(conf, mpp);
336         select_deferred_remove(conf, mpp);
337         select_marginal_path_err_sample_time(conf, mpp);
338         select_marginal_path_err_rate_threshold(conf, mpp);
339         select_marginal_path_err_recheck_gap_time(conf, mpp);
340         select_marginal_path_double_failed_time(conf, mpp);
341         select_san_path_err_threshold(conf, mpp);
342         select_san_path_err_forget_rate(conf, mpp);
343         select_san_path_err_recovery_time(conf, mpp);
344         select_delay_checks(conf, mpp);
345         select_skip_kpartx(conf, mpp);
346         select_max_sectors_kb(conf, mpp);
347         select_ghost_delay(conf, mpp);
348         select_flush_on_last_del(conf, mpp);
349
350         sysfs_set_scsi_tmo(mpp, conf->checkint);
351         marginal_pathgroups = conf->marginal_pathgroups;
352         pthread_cleanup_pop(1);
353
354         if (!mpp->features || !mpp->hwhandler || !mpp->selector) {
355                 condlog(0, "%s: map select failed", mpp->alias);
356                 return 1;
357         }
358
359         if (marginal_path_check_enabled(mpp))
360                 start_io_err_stat_thread(vecs);
361
362         /*
363          * assign paths to path groups -- start with no groups and all paths
364          * in mpp->paths
365          */
366         if (mpp->pg) {
367                 vector_foreach_slot (mpp->pg, pgp, i)
368                         free_pathgroup(pgp, KEEP_PATHS);
369
370                 vector_free(mpp->pg);
371                 mpp->pg = NULL;
372         }
373         if (group_paths(mpp, marginal_pathgroups))
374                 return 1;
375
376         /*
377          * ponders each path group and determine highest prio pg
378          * to switch over (default to first)
379          */
380         mpp->bestpg = select_path_group(mpp);
381
382         /* re-order paths in all path groups in an optimized way
383          * for round-robin path selectors to get maximum throughput.
384          */
385         if (!strncmp(mpp->selector, "round-robin", 11)) {
386                 vector_foreach_slot(mpp->pg, pgp, i) {
387                         if (VECTOR_SIZE(pgp->paths) <= 2)
388                                 continue;
389                         if (rr_optimize_path_order(pgp)) {
390                                 condlog(2, "cannot re-order paths for "
391                                         "optimization: %s",
392                                         mpp->alias);
393                                 return 1;
394                         }
395                 }
396         }
397
398         /*
399          * transform the mp->pg vector of vectors of paths
400          * into a mp->params strings to feed the device-mapper
401          */
402         if (assemble_map(mpp, params)) {
403                 condlog(0, "%s: problem assembing map", mpp->alias);
404                 return 1;
405         }
406         return 0;
407 }
408
409 static void
410 compute_pgid(struct pathgroup * pgp)
411 {
412         struct path * pp;
413         int i;
414
415         vector_foreach_slot (pgp->paths, pp, i)
416                 pgp->id ^= (long)pp;
417 }
418
419 static int
420 pgcmp (struct multipath * mpp, struct multipath * cmpp)
421 {
422         int i, j;
423         struct pathgroup * pgp;
424         struct pathgroup * cpgp;
425         int r = 0;
426
427         if (!mpp)
428                 return 0;
429
430         vector_foreach_slot (mpp->pg, pgp, i) {
431                 compute_pgid(pgp);
432
433                 vector_foreach_slot (cmpp->pg, cpgp, j) {
434                         if (pgp->id == cpgp->id &&
435                             !pathcmp(pgp, cpgp)) {
436                                 r = 0;
437                                 break;
438                         }
439                         r++;
440                 }
441                 if (r)
442                         return r;
443         }
444         return r;
445 }
446
447 static struct udev_device *
448 get_udev_for_mpp(const struct multipath *mpp)
449 {
450         dev_t devnum;
451         struct udev_device *udd;
452
453         if (!mpp || !mpp->dmi) {
454                 condlog(1, "%s called with empty mpp", __func__);
455                 return NULL;
456         }
457
458         devnum = makedev(mpp->dmi->major, mpp->dmi->minor);
459         udd = udev_device_new_from_devnum(udev, 'b', devnum);
460         if (!udd) {
461                 condlog(1, "failed to get udev device for %s", mpp->alias);
462                 return NULL;
463         }
464         return udd;
465 }
466
467 void trigger_partitions_udev_change(struct udev_device *dev,
468                                     const char *action, int len)
469 {
470         struct udev_enumerate *part_enum;
471         struct udev_list_entry *item;
472         const char *devtype;
473
474         part_enum = udev_enumerate_new(udev);
475         if (!part_enum)
476                 return;
477
478         if (udev_enumerate_add_match_parent(part_enum, dev) < 0 ||
479             udev_enumerate_add_match_subsystem(part_enum, "block") < 0 ||
480             udev_enumerate_scan_devices(part_enum) < 0)
481                 goto unref;
482
483         udev_list_entry_foreach(item,
484                                 udev_enumerate_get_list_entry(part_enum)) {
485                 const char *syspath;
486                 struct udev_device *part;
487
488                 syspath = udev_list_entry_get_name(item);
489                 part = udev_device_new_from_syspath(udev, syspath);
490                 if (!part)
491                         continue;
492
493                 devtype = udev_device_get_devtype(part);
494                 if (devtype && !strcmp("partition", devtype)) {
495                         condlog(4, "%s: triggering %s event for %s", __func__,
496                                 action, syspath);
497                         sysfs_attr_set_value(part, "uevent", action, len);
498                 }
499                 udev_device_unref(part);
500         }
501 unref:
502         udev_enumerate_unref(part_enum);
503 }
504
505 void
506 trigger_paths_udev_change(struct multipath *mpp, bool is_mpath)
507 {
508         struct pathgroup *pgp;
509         struct path *pp;
510         int i, j;
511         /*
512          * If a path changes from multipath to non-multipath, we must
513          * synthesize an artificial "add" event, otherwise the LVM2 rules
514          * (69-lvm2-lvmetad.rules) won't pick it up. Otherwise, we'd just
515          * irritate ourselves with an "add", so use "change".
516          */
517         const char *action = is_mpath ? "change" : "add";
518
519         if (!mpp || !mpp->pg)
520                 return;
521
522         vector_foreach_slot (mpp->pg, pgp, i) {
523                 if (!pgp->paths)
524                         continue;
525                 vector_foreach_slot(pgp->paths, pp, j) {
526                         const char *env;
527
528                         if (!pp->udev)
529                                 continue;
530                         /*
531                          * Paths that are already classified as multipath
532                          * members don't need another uevent.
533                          */
534                         env = udev_device_get_property_value(
535                                 pp->udev, "DM_MULTIPATH_DEVICE_PATH");
536
537                         if (is_mpath && env != NULL && !strcmp(env, "1")) {
538                                 /*
539                                  * If FIND_MULTIPATHS_WAIT_UNTIL is not "0",
540                                  * path is in "maybe" state and timer is running
541                                  * Send uevent now (see multipath.rules).
542                                  */
543                                 env = udev_device_get_property_value(
544                                         pp->udev, "FIND_MULTIPATHS_WAIT_UNTIL");
545                                 if (env == NULL || !strcmp(env, "0"))
546                                         continue;
547                         } else if (!is_mpath &&
548                                    (env == NULL || !strcmp(env, "0")))
549                                 continue;
550
551                         condlog(3, "triggering %s uevent for %s (is %smultipath member)",
552                                 action, pp->dev, is_mpath ? "" : "no ");
553                         sysfs_attr_set_value(pp->udev, "uevent",
554                                              action, strlen(action));
555                         trigger_partitions_udev_change(pp->udev, action,
556                                                        strlen(action));
557                 }
558         }
559
560         mpp->needs_paths_uevent = 0;
561 }
562
563 static int
564 sysfs_set_max_sectors_kb(struct multipath *mpp, int is_reload)
565 {
566         struct pathgroup * pgp;
567         struct path *pp;
568         char buff[11];
569         int i, j, ret, err = 0;
570         struct udev_device *udd;
571         int max_sectors_kb;
572
573         if (mpp->max_sectors_kb == MAX_SECTORS_KB_UNDEF)
574                 return 0;
575         max_sectors_kb = mpp->max_sectors_kb;
576         if (is_reload) {
577                 if (!mpp->dmi && dm_get_info(mpp->alias, &mpp->dmi) != 0) {
578                         condlog(1, "failed to get dm info for %s", mpp->alias);
579                         return 1;
580                 }
581                 udd = get_udev_for_mpp(mpp);
582                 if (!udd) {
583                         condlog(1, "failed to get udev device to set max_sectors_kb for %s", mpp->alias);
584                         return 1;
585                 }
586                 ret = sysfs_attr_get_value(udd, "queue/max_sectors_kb", buff,
587                                            sizeof(buff));
588                 udev_device_unref(udd);
589                 if (ret <= 0) {
590                         condlog(1, "failed to get current max_sectors_kb from %s", mpp->alias);
591                         return 1;
592                 }
593                 if (sscanf(buff, "%u\n", &max_sectors_kb) != 1) {
594                         condlog(1, "can't parse current max_sectors_kb from %s",
595                                 mpp->alias);
596                         return 1;
597                 }
598         }
599         snprintf(buff, 11, "%d", max_sectors_kb);
600
601         vector_foreach_slot (mpp->pg, pgp, i) {
602                 vector_foreach_slot(pgp->paths, pp, j) {
603                         ret = sysfs_attr_set_value(pp->udev,
604                                                    "queue/max_sectors_kb",
605                                                    buff, strlen(buff));
606                         if (ret < 0) {
607                                 condlog(1, "failed setting max_sectors_kb on %s : %s", pp->dev, strerror(-ret));
608                                 err = 1;
609                         }
610                 }
611         }
612         return err;
613 }
614
615 static bool is_udev_ready(struct multipath *cmpp)
616 {
617         struct udev_device *mpp_ud;
618         const char *env;
619         bool rc;
620
621         /*
622          * MPATH_DEVICE_READY != 1 can mean two things:
623          *  (a) no usable paths
624          *  (b) device was never fully processed (e.g. udev killed)
625          * If we are in this code path (startup or forced reconfigure),
626          * (b) can mean that upper layers like kpartx have never been
627          * run for this map. Thus force udev reload.
628          */
629
630         mpp_ud = get_udev_for_mpp(cmpp);
631         if (!mpp_ud)
632                 return true;
633         env = udev_device_get_property_value(mpp_ud, "MPATH_DEVICE_READY");
634         rc = (env != NULL && !strcmp(env, "1"));
635         udev_device_unref(mpp_ud);
636         condlog(4, "%s: %s: \"%s\" -> %d\n", __func__, cmpp->alias, env, rc);
637         return rc;
638 }
639
640 static void
641 select_reload_action(struct multipath *mpp, const char *reason)
642 {
643         mpp->action = ACT_RELOAD;
644         condlog(3, "%s: set ACT_RELOAD (%s)", mpp->alias, reason);
645 }
646
647 void select_action (struct multipath *mpp, const struct _vector *curmp,
648                     int force_reload)
649 {
650         struct multipath * cmpp;
651         struct multipath * cmpp_by_name;
652         char * mpp_feat, * cmpp_feat;
653
654         cmpp = find_mp_by_wwid(curmp, mpp->wwid);
655         cmpp_by_name = find_mp_by_alias(curmp, mpp->alias);
656         if (mpp->need_reload || (cmpp && cmpp->need_reload))
657                 force_reload = 1;
658
659         if (!cmpp_by_name) {
660                 if (cmpp) {
661                         condlog(2, "%s: rename %s to %s", mpp->wwid,
662                                 cmpp->alias, mpp->alias);
663                         strlcpy(mpp->alias_old, cmpp->alias, WWID_SIZE);
664                         mpp->action = ACT_RENAME;
665                         if (force_reload) {
666                                 mpp->force_udev_reload = 1;
667                                 mpp->action = ACT_FORCERENAME;
668                         }
669                         return;
670                 }
671                 mpp->action = ACT_CREATE;
672                 condlog(3, "%s: set ACT_CREATE (map does not exist)",
673                         mpp->alias);
674                 return;
675         }
676
677         if (!cmpp) {
678                 condlog(1, "%s: can't use alias \"%s\" used by %s, falling back to WWID",
679                         mpp->wwid, mpp->alias, cmpp_by_name->wwid);
680                 /* We can do this because wwid wasn't found */
681                 free(mpp->alias);
682                 mpp->alias = strdup(mpp->wwid);
683                 mpp->action = ACT_CREATE;
684                 condlog(3, "%s: set ACT_CREATE (map does not exist, name changed)",
685                         mpp->alias);
686                 return;
687         }
688
689         if (cmpp != cmpp_by_name) {
690                 condlog(2, "%s: unable to rename %s to %s (%s is used by %s)",
691                         mpp->wwid, cmpp->alias, mpp->alias,
692                         mpp->alias, cmpp_by_name->wwid);
693                 /* reset alias to existing alias */
694                 free(mpp->alias);
695                 mpp->alias = strdup(cmpp->alias);
696                 mpp->action = ACT_IMPOSSIBLE;
697                 return;
698         }
699
700         if (force_reload) {
701                 mpp->force_udev_reload = 1;
702                 mpp->action = ACT_RELOAD;
703                 condlog(3, "%s: set ACT_RELOAD (forced by user)",
704                         mpp->alias);
705                 return;
706         }
707         if (cmpp->size != mpp->size) {
708                 mpp->force_udev_reload = 1;
709                 mpp->action = ACT_RESIZE;
710                 condlog(3, "%s: set ACT_RESIZE (size change)",
711                         mpp->alias);
712                 return;
713         }
714
715         if (!is_udev_ready(cmpp) && count_active_paths(mpp) > 0) {
716                 mpp->force_udev_reload = 1;
717                 mpp->action = ACT_RELOAD;
718                 condlog(3, "%s: set ACT_RELOAD (udev incomplete)",
719                         mpp->alias);
720                 return;
721         }
722
723         if (mpp->no_path_retry != NO_PATH_RETRY_UNDEF &&
724             !!strstr(mpp->features, "queue_if_no_path") !=
725             !!strstr(cmpp->features, "queue_if_no_path")) {
726                 select_reload_action(mpp, "no_path_retry change");
727                 return;
728         }
729         if ((mpp->retain_hwhandler != RETAIN_HWHANDLER_ON ||
730              strcmp(cmpp->hwhandler, "0") == 0) &&
731             (strlen(cmpp->hwhandler) != strlen(mpp->hwhandler) ||
732              strncmp(cmpp->hwhandler, mpp->hwhandler,
733                     strlen(mpp->hwhandler)))) {
734                 select_reload_action(mpp, "hwhandler change");
735                 return;
736         }
737
738         if (mpp->retain_hwhandler != RETAIN_HWHANDLER_UNDEF &&
739             !!strstr(mpp->features, "retain_attached_hw_handler") !=
740             !!strstr(cmpp->features, "retain_attached_hw_handler") &&
741             get_linux_version_code() < KERNEL_VERSION(4, 3, 0)) {
742                 select_reload_action(mpp, "retain_hwhandler change");
743                 return;
744         }
745
746         cmpp_feat = strdup(cmpp->features);
747         mpp_feat = strdup(mpp->features);
748         if (cmpp_feat && mpp_feat) {
749                 remove_feature(&mpp_feat, "queue_if_no_path");
750                 remove_feature(&mpp_feat, "retain_attached_hw_handler");
751                 remove_feature(&cmpp_feat, "queue_if_no_path");
752                 remove_feature(&cmpp_feat, "retain_attached_hw_handler");
753                 if (strcmp(mpp_feat, cmpp_feat)) {
754                         select_reload_action(mpp, "features change");
755                         free(cmpp_feat);
756                         free(mpp_feat);
757                         return;
758                 }
759         }
760         free(cmpp_feat);
761         free(mpp_feat);
762
763         if (!cmpp->selector || strncmp(cmpp->selector, mpp->selector,
764                     strlen(mpp->selector))) {
765                 select_reload_action(mpp, "selector change");
766                 return;
767         }
768         if (cmpp->minio != mpp->minio) {
769                 select_reload_action(mpp, "minio change");
770                 return;
771         }
772         if (!cmpp->pg || VECTOR_SIZE(cmpp->pg) != VECTOR_SIZE(mpp->pg)) {
773                 select_reload_action(mpp, "path group number change");
774                 return;
775         }
776         if (pgcmp(mpp, cmpp)) {
777                 select_reload_action(mpp, "path group topology change");
778                 return;
779         }
780         if (cmpp->nextpg != mpp->bestpg) {
781                 mpp->action = ACT_SWITCHPG;
782                 condlog(3, "%s: set ACT_SWITCHPG (next path group change)",
783                         mpp->alias);
784                 return;
785         }
786         mpp->action = ACT_NOTHING;
787         condlog(3, "%s: set ACT_NOTHING (map unchanged)",
788                 mpp->alias);
789         return;
790 }
791
792 int reinstate_paths(struct multipath *mpp)
793 {
794         int i, j;
795         struct pathgroup * pgp;
796         struct path * pp;
797
798         if (!mpp->pg)
799                 return 0;
800
801         vector_foreach_slot (mpp->pg, pgp, i) {
802                 if (!pgp->paths)
803                         continue;
804
805                 vector_foreach_slot (pgp->paths, pp, j) {
806                         if (pp->state != PATH_UP &&
807                             (pgp->status == PGSTATE_DISABLED ||
808                              pgp->status == PGSTATE_ACTIVE))
809                                 continue;
810
811                         if (pp->dmstate == PSTATE_FAILED) {
812                                 if (dm_reinstate_path(mpp->alias, pp->dev_t))
813                                         condlog(0, "%s: error reinstating",
814                                                 pp->dev);
815                         }
816                 }
817         }
818         return 0;
819 }
820
821 static int
822 lock_multipath (struct multipath * mpp, int lock)
823 {
824         struct pathgroup * pgp;
825         struct path * pp;
826         int i, j;
827         int x, y;
828
829         if (!mpp || !mpp->pg)
830                 return 0;
831
832         vector_foreach_slot (mpp->pg, pgp, i) {
833                 if (!pgp->paths)
834                         continue;
835                 vector_foreach_slot(pgp->paths, pp, j) {
836                         if (lock && flock(pp->fd, LOCK_SH | LOCK_NB) &&
837                             errno == EWOULDBLOCK)
838                                 goto fail;
839                         else if (!lock)
840                                 flock(pp->fd, LOCK_UN);
841                 }
842         }
843         return 0;
844 fail:
845         vector_foreach_slot (mpp->pg, pgp, x) {
846                 if (x > i)
847                         return 1;
848                 if (!pgp->paths)
849                         continue;
850                 vector_foreach_slot(pgp->paths, pp, y) {
851                         if (x == i && y >= j)
852                                 return 1;
853                         flock(pp->fd, LOCK_UN);
854                 }
855         }
856         return 1;
857 }
858
859 int domap(struct multipath *mpp, char *params, int is_daemon)
860 {
861         int r = DOMAP_FAIL;
862         struct config *conf;
863
864         /*
865          * last chance to quit before touching the devmaps
866          */
867         if (mpp->action == ACT_DRY_RUN) {
868                 print_multipath_topology(mpp, libmp_verbosity);
869                 return DOMAP_DRY;
870         }
871
872         if (mpp->action == ACT_CREATE && dm_map_present(mpp->alias)) {
873                 char wwid[WWID_SIZE];
874
875                 if (dm_get_uuid(mpp->alias, wwid, sizeof(wwid)) == 0) {
876                         if (!strncmp(mpp->wwid, wwid, sizeof(wwid))) {
877                                 condlog(3, "%s: map already present",
878                                         mpp->alias);
879                                 mpp->action = ACT_RELOAD;
880                         } else {
881                                 condlog(0, "%s: map \"%s\" already present with WWID %s, skipping",
882                                         mpp->wwid, mpp->alias, wwid);
883                                 condlog(0, "please check alias settings in config and bindings file");
884                                 mpp->action = ACT_REJECT;
885                         }
886                 }
887         }
888
889         switch (mpp->action) {
890         case ACT_REJECT:
891         case ACT_NOTHING:
892         case ACT_IMPOSSIBLE:
893                 return DOMAP_EXIST;
894
895         case ACT_SWITCHPG:
896                 dm_switchgroup(mpp->alias, mpp->bestpg);
897                 /*
898                  * we may have avoided reinstating paths because there where in
899                  * active or disabled PG. Now that the topology has changed,
900                  * retry.
901                  */
902                 reinstate_paths(mpp);
903                 return DOMAP_EXIST;
904
905         case ACT_CREATE:
906                 if (lock_multipath(mpp, 1)) {
907                         condlog(3, "%s: failed to create map (in use)",
908                                 mpp->alias);
909                         return DOMAP_RETRY;
910                 }
911
912                 sysfs_set_max_sectors_kb(mpp, 0);
913                 if (is_daemon && mpp->ghost_delay > 0 && count_active_paths(mpp) &&
914                     pathcount(mpp, PATH_UP) == 0)
915                         mpp->ghost_delay_tick = mpp->ghost_delay;
916                 r = dm_addmap_create(mpp, params);
917
918                 lock_multipath(mpp, 0);
919                 break;
920
921         case ACT_RELOAD:
922                 sysfs_set_max_sectors_kb(mpp, 1);
923                 if (mpp->ghost_delay_tick > 0 && pathcount(mpp, PATH_UP))
924                         mpp->ghost_delay_tick = 0;
925                 r = dm_addmap_reload(mpp, params, 0);
926                 break;
927
928         case ACT_RESIZE:
929                 sysfs_set_max_sectors_kb(mpp, 1);
930                 if (mpp->ghost_delay_tick > 0 && pathcount(mpp, PATH_UP))
931                         mpp->ghost_delay_tick = 0;
932                 r = dm_addmap_reload(mpp, params, 1);
933                 break;
934
935         case ACT_RENAME:
936                 conf = get_multipath_config();
937                 pthread_cleanup_push(put_multipath_config, conf);
938                 r = dm_rename(mpp->alias_old, mpp->alias,
939                               conf->partition_delim, mpp->skip_kpartx);
940                 pthread_cleanup_pop(1);
941                 break;
942
943         case ACT_FORCERENAME:
944                 conf = get_multipath_config();
945                 pthread_cleanup_push(put_multipath_config, conf);
946                 r = dm_rename(mpp->alias_old, mpp->alias,
947                               conf->partition_delim, mpp->skip_kpartx);
948                 pthread_cleanup_pop(1);
949                 if (r) {
950                         sysfs_set_max_sectors_kb(mpp, 1);
951                         if (mpp->ghost_delay_tick > 0 &&
952                             pathcount(mpp, PATH_UP))
953                                 mpp->ghost_delay_tick = 0;
954                         r = dm_addmap_reload(mpp, params, 0);
955                 }
956                 break;
957
958         default:
959                 break;
960         }
961
962         if (r == DOMAP_OK) {
963                 /*
964                  * DM_DEVICE_CREATE, DM_DEVICE_RENAME, or DM_DEVICE_RELOAD
965                  * succeeded
966                  */
967                 mpp->force_udev_reload = 0;
968                 if (mpp->action == ACT_CREATE &&
969                     (remember_wwid(mpp->wwid) == 1 ||
970                      mpp->needs_paths_uevent))
971                         trigger_paths_udev_change(mpp, true);
972                 if (!is_daemon) {
973                         /* multipath client mode */
974                         dm_switchgroup(mpp->alias, mpp->bestpg);
975                 } else  {
976                         /* multipath daemon mode */
977                         mpp->stat_map_loads++;
978                         condlog(4, "%s: load table [0 %llu %s %s]", mpp->alias,
979                                 mpp->size, TGT_MPATH, params);
980                         /*
981                          * Required action is over, reset for the stateful daemon.
982                          * But don't do it for creation as we use in the caller the
983                          * mpp->action to figure out whether to start the watievent checker.
984                          */
985                         if (mpp->action != ACT_CREATE)
986                                 mpp->action = ACT_NOTHING;
987                         else {
988                                 conf = get_multipath_config();
989                                 mpp->wait_for_udev = 1;
990                                 mpp->uev_wait_tick = conf->uev_wait_timeout;
991                                 put_multipath_config(conf);
992                         }
993                 }
994                 dm_setgeometry(mpp);
995                 return DOMAP_OK;
996         } else if (r == DOMAP_FAIL && mpp->action == ACT_CREATE &&
997                    mpp->needs_paths_uevent)
998                 trigger_paths_udev_change(mpp, false);
999
1000         return DOMAP_FAIL;
1001 }
1002
1003 extern int
1004 check_daemon(void)
1005 {
1006         int fd;
1007         char *reply;
1008         int ret = 0;
1009         unsigned int timeout;
1010         struct config *conf;
1011
1012         fd = mpath_connect();
1013         if (fd == -1)
1014                 return 0;
1015
1016         if (send_packet(fd, "show daemon") != 0)
1017                 goto out;
1018         conf = get_multipath_config();
1019         timeout = conf->uxsock_timeout;
1020         put_multipath_config(conf);
1021         if (recv_packet(fd, &reply, timeout) != 0)
1022                 goto out;
1023
1024         if (reply && strstr(reply, "shutdown"))
1025                 goto out_free;
1026
1027         ret = 1;
1028
1029 out_free:
1030         free(reply);
1031 out:
1032         mpath_disconnect(fd);
1033         return ret;
1034 }
1035
1036 /*
1037  * The force_reload parameter determines how coalesce_paths treats existing maps.
1038  * FORCE_RELOAD_NONE: existing maps aren't touched at all
1039  * FORCE_RELOAD_YES: all maps are rebuilt from scratch and (re)loaded in DM
1040  * FORCE_RELOAD_WEAK: existing maps are compared to the current conf and only
1041  * reloaded in DM if there's a difference. This is normally sufficient.
1042  */
1043 int coalesce_paths (struct vectors *vecs, vector mpvec, char *refwwid,
1044                     int force_reload, enum mpath_cmds cmd)
1045 {
1046         int ret = CP_FAIL;
1047         int k, i, r;
1048         int is_daemon = (cmd == CMD_NONE) ? 1 : 0;
1049         char *params __attribute__((cleanup(cleanup_charp))) = NULL;
1050         struct multipath * mpp;
1051         struct path * pp1 = NULL;
1052         struct path * pp2;
1053         vector curmp = vecs->mpvec;
1054         vector pathvec = vecs->pathvec;
1055         vector newmp;
1056         struct config *conf = NULL;
1057         int allow_queueing;
1058         struct bitfield *size_mismatch_seen;
1059
1060         /* ignore refwwid if it's empty */
1061         if (refwwid && !strlen(refwwid))
1062                 refwwid = NULL;
1063
1064         if (force_reload != FORCE_RELOAD_NONE) {
1065                 vector_foreach_slot (pathvec, pp1, k) {
1066                         pp1->mpp = NULL;
1067                 }
1068         }
1069
1070         if (VECTOR_SIZE(pathvec) == 0)
1071                 return CP_OK;
1072         size_mismatch_seen = alloc_bitfield(VECTOR_SIZE(pathvec));
1073         if (size_mismatch_seen == NULL)
1074                 return CP_FAIL;
1075
1076         if (mpvec)
1077                 newmp = mpvec;
1078         else
1079                 newmp = vector_alloc();
1080         if (!newmp) {
1081                 condlog(0, "can not allocate newmp");
1082                 goto out;
1083         }
1084
1085         vector_foreach_slot (pathvec, pp1, k) {
1086                 int invalid;
1087
1088                 if (should_exit()) {
1089                         ret = CP_FAIL;
1090                         goto out;
1091                 }
1092
1093                 /* skip this path for some reason */
1094
1095                 /* 1. if path has no unique id or wwid blacklisted */
1096                 if (strlen(pp1->wwid) == 0) {
1097                         orphan_path(pp1, "no WWID");
1098                         continue;
1099                 }
1100
1101                 conf = get_multipath_config();
1102                 pthread_cleanup_push(put_multipath_config, conf);
1103                 invalid = (filter_path(conf, pp1) > 0);
1104                 pthread_cleanup_pop(1);
1105                 if (invalid) {
1106                         orphan_path(pp1, "blacklisted");
1107                         continue;
1108                 }
1109
1110                 /* 2. if path already coalesced, or seen and discarded */
1111                 if (pp1->mpp || is_bit_set_in_bitfield(k, size_mismatch_seen))
1112                         continue;
1113
1114                 /* 3. if path has disappeared */
1115                 if (pp1->state == PATH_REMOVED) {
1116                         orphan_path(pp1, "path removed");
1117                         continue;
1118                 }
1119
1120                 /* 4. path is out of scope */
1121                 if (refwwid && strncmp(pp1->wwid, refwwid, WWID_SIZE - 1))
1122                         continue;
1123
1124                 /* If find_multipaths was selected check if the path is valid */
1125                 if (!refwwid && !should_multipath(pp1, pathvec, curmp)) {
1126                         orphan_path(pp1, "only one path");
1127                         continue;
1128                 }
1129
1130                 /*
1131                  * at this point, we know we really got a new mp
1132                  */
1133                 mpp = add_map_with_path(vecs, pp1, 0);
1134                 if (!mpp) {
1135                         orphan_path(pp1, "failed to create multipath device");
1136                         continue;
1137                 }
1138
1139                 if (!mpp->paths) {
1140                         condlog(0, "%s: skip coalesce (no paths)", mpp->alias);
1141                         remove_map(mpp, vecs->pathvec, NULL);
1142                         continue;
1143                 }
1144
1145                 for (i = k + 1; i < VECTOR_SIZE(pathvec); i++) {
1146                         pp2 = VECTOR_SLOT(pathvec, i);
1147
1148                         if (strcmp(pp1->wwid, pp2->wwid))
1149                                 continue;
1150
1151                         if (!mpp->size && pp2->size)
1152                                 mpp->size = pp2->size;
1153
1154                         if (mpp->size && pp2->size &&
1155                             pp2->size != mpp->size) {
1156                                 /*
1157                                  * ouch, avoid feeding that to the DM
1158                                  */
1159                                 condlog(0, "%s: size %llu, expected %llu. "
1160                                         "Discard", pp2->dev, pp2->size,
1161                                         mpp->size);
1162                                 mpp->action = ACT_REJECT;
1163                                 set_bit_in_bitfield(i, size_mismatch_seen);
1164                         }
1165                 }
1166                 verify_paths(mpp);
1167
1168                 if (setup_map(mpp, &params, vecs)) {
1169                         remove_map(mpp, vecs->pathvec, NULL);
1170                         continue;
1171                 }
1172
1173                 if (cmd == CMD_DRY_RUN)
1174                         mpp->action = ACT_DRY_RUN;
1175                 if (mpp->action == ACT_UNDEF)
1176                         select_action(mpp, curmp,
1177                                       force_reload == FORCE_RELOAD_YES ? 1 : 0);
1178
1179                 r = domap(mpp, params, is_daemon);
1180                 free(params);
1181                 params = NULL;
1182
1183                 if (r == DOMAP_FAIL || r == DOMAP_RETRY) {
1184                         condlog(3, "%s: domap (%u) failure "
1185                                    "for create/reload map",
1186                                 mpp->alias, r);
1187                         if (r == DOMAP_FAIL || is_daemon) {
1188                                 condlog(2, "%s: %s map",
1189                                         mpp->alias, (mpp->action == ACT_CREATE)?
1190                                         "ignoring" : "removing");
1191                                 remove_map(mpp, vecs->pathvec, NULL);
1192                                 continue;
1193                         } else /* if (r == DOMAP_RETRY && !is_daemon) */ {
1194                                 ret = CP_RETRY;
1195                                 goto out;
1196                         }
1197                 }
1198                 if (r == DOMAP_DRY) {
1199                         if (!vector_alloc_slot(newmp)) {
1200                                 remove_map(mpp, vecs->pathvec, NULL);
1201                                 goto out;
1202                         }
1203                         vector_set_slot(newmp, mpp);
1204                         continue;
1205                 }
1206
1207                 conf = get_multipath_config();
1208                 allow_queueing = conf->allow_queueing;
1209                 put_multipath_config(conf);
1210                 if (!is_daemon && !allow_queueing && !check_daemon()) {
1211                         if (mpp->no_path_retry != NO_PATH_RETRY_UNDEF &&
1212                             mpp->no_path_retry != NO_PATH_RETRY_FAIL)
1213                                 condlog(3, "%s: multipathd not running, unset "
1214                                         "queue_if_no_path feature", mpp->alias);
1215                         if (!dm_queue_if_no_path(mpp->alias, 0))
1216                                 remove_feature(&mpp->features,
1217                                                "queue_if_no_path");
1218                 }
1219
1220                 if (!is_daemon && mpp->action != ACT_NOTHING)
1221                         print_multipath_topology(mpp, libmp_verbosity);
1222
1223                 if (mpp->action != ACT_REJECT) {
1224                         if (!vector_alloc_slot(newmp)) {
1225                                 remove_map(mpp, vecs->pathvec, NULL);
1226                                 goto out;
1227                         }
1228                         vector_set_slot(newmp, mpp);
1229                 }
1230                 else
1231                         remove_map(mpp, vecs->pathvec, NULL);
1232         }
1233         ret = CP_OK;
1234 out:
1235         free(size_mismatch_seen);
1236         if (!mpvec)
1237                 free_multipathvec(newmp, KEEP_PATHS);
1238         return ret;
1239 }
1240
1241 struct udev_device *get_udev_device(const char *dev, enum devtypes dev_type)
1242 {
1243         struct udev_device *ud = NULL;
1244         const char *base;
1245
1246         if (dev == NULL || *dev == '\0')
1247                 return NULL;
1248
1249         switch (dev_type) {
1250         case DEV_DEVNODE:
1251         case DEV_DEVMAP:
1252                 /* This should be GNU basename, compiler will warn if not */
1253                 base = basename(dev);
1254                 if (*base == '\0')
1255                         break;
1256                 ud = udev_device_new_from_subsystem_sysname(udev, "block",
1257                                                             base);
1258                 break;
1259         case DEV_DEVT:
1260                 ud = udev_device_new_from_devnum(udev, 'b', parse_devt(dev));
1261                 break;
1262         case DEV_UEVENT:
1263                 ud = udev_device_new_from_environment(udev);
1264                 break;
1265         default:
1266                 condlog(0, "Internal error: get_udev_device called with invalid type %d\n",
1267                         dev_type);
1268                 break;
1269         }
1270         if (ud == NULL)
1271                 condlog(2, "get_udev_device: failed to look up %s with type %d",
1272                         dev, dev_type);
1273         return ud;
1274 }
1275
1276 static int _get_refwwid(enum mpath_cmds cmd, const char *dev,
1277                         enum devtypes dev_type,
1278                         vector pathvec, struct config *conf, char **wwid)
1279 {
1280         int ret = 1;
1281         struct path * pp;
1282         char buff[FILE_NAME_SIZE];
1283         const char *refwwid = NULL;
1284         char tmpwwid[WWID_SIZE];
1285         struct udev_device *udevice;
1286         int flags = DI_SYSFS | DI_WWID;
1287
1288         if (!wwid)
1289                 return PATHINFO_FAILED;
1290         *wwid = NULL;
1291
1292         if (dev_type == DEV_NONE)
1293                 return PATHINFO_FAILED;
1294
1295         if (cmd != CMD_REMOVE_WWID)
1296                 flags |= DI_BLACKLIST;
1297
1298         switch (dev_type) {
1299         case DEV_DEVNODE:
1300                 if (basenamecpy(dev, buff, FILE_NAME_SIZE) == 0) {
1301                         condlog(1, "basename failed for '%s' (%s)",
1302                                 dev, buff);
1303                         return PATHINFO_FAILED;
1304                 }
1305
1306                 /* dev is used in common code below */
1307                 dev = buff;
1308                 pp = find_path_by_dev(pathvec, dev);
1309                 goto common;
1310
1311         case DEV_DEVT:
1312                 pp = find_path_by_devt(pathvec, dev);
1313                 goto common;
1314
1315         case DEV_UEVENT:
1316                 pp = NULL;
1317                 /* For condlog below, dev is unused in get_udev_device() */
1318                 dev = "environment";
1319         common:
1320                 if (!pp) {
1321                         udevice = get_udev_device(dev, dev_type);
1322
1323                         if (!udevice) {
1324                                 condlog(0, "%s: cannot find block device", dev);
1325                                 return PATHINFO_FAILED;
1326                         }
1327
1328                         ret = store_pathinfo(pathvec, conf, udevice,
1329                                              flags, &pp);
1330                         udev_device_unref(udevice);
1331                         if (!pp) {
1332                                 if (ret == PATHINFO_FAILED)
1333                                         condlog(0, "%s: can't store path info",
1334                                                 dev);
1335                                 return ret;
1336                         }
1337                 }
1338                 if (flags & DI_BLACKLIST &&
1339                     filter_property(conf, pp->udev, 3, pp->uid_attribute) > 0)
1340                         return PATHINFO_SKIPPED;
1341                 refwwid = pp->wwid;
1342                 break;
1343
1344         case DEV_DEVMAP:
1345                 if (((dm_get_uuid(dev, tmpwwid, WWID_SIZE)) == 0)
1346                     && (strlen(tmpwwid)))
1347                         refwwid = tmpwwid;
1348
1349                 /* or may be a binding */
1350                 else if (get_user_friendly_wwid(dev, tmpwwid,
1351                                                 conf->bindings_file) == 0)
1352                         refwwid = tmpwwid;
1353
1354                 /* or may be an alias */
1355                 else {
1356                         refwwid = get_mpe_wwid(conf->mptable, dev);
1357
1358                         /* or directly a wwid */
1359                         if (!refwwid)
1360                                 refwwid = dev;
1361                 }
1362
1363                 if (flags & DI_BLACKLIST && refwwid && strlen(refwwid) &&
1364                     filter_wwid(conf->blist_wwid, conf->elist_wwid, refwwid,
1365                                 NULL) > 0)
1366                         return PATHINFO_SKIPPED;
1367                 break;
1368         default:
1369                 break;
1370         }
1371
1372         if (refwwid && strlen(refwwid)) {
1373                 *wwid = strdup(refwwid);
1374                 return PATHINFO_OK;
1375         }
1376
1377         return PATHINFO_FAILED;
1378 }
1379
1380 /*
1381  * Returns: PATHINFO_OK, PATHINFO_FAILED, or PATHINFO_SKIPPED (see pathinfo())
1382  */
1383 int get_refwwid(enum mpath_cmds cmd, const char *dev, enum devtypes dev_type,
1384                 vector pathvec, char **wwid)
1385
1386 {
1387         int ret;
1388         struct config *conf = get_multipath_config();
1389
1390         pthread_cleanup_push(put_multipath_config, conf);
1391         ret = _get_refwwid(cmd, dev, dev_type, pathvec, conf, wwid);
1392         pthread_cleanup_pop(1);
1393         return ret;
1394 }