2 * Copyright (C) 2007-2009 Red Hat, Inc. All rights reserved.
4 * This file is part of LVM2.
6 * This copyrighted material is made available to anyone wishing to use,
7 * modify, copy, or redistribute it subject to the terms and conditions
8 * of the GNU Lesser General Public License v.2.1.
10 * You should have received a copy of the GNU Lesser General Public License
11 * along with this program; if not, write to the Free Software Foundation,
12 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 * This provides the interface between clvmd and OpenAIS as the cluster
20 #include "clvmd-common.h"
26 #include <openais/saAis.h>
27 #include <openais/saLck.h>
29 #include <corosync/corotypes.h>
30 #include <corosync/cpg.h>
34 #include "clvmd-comms.h"
35 #include "lvm-functions.h"
38 /* Timeout value for several openais calls */
41 static void openais_cpg_deliver_callback (cpg_handle_t handle,
42 const struct cpg_name *groupName,
47 static void openais_cpg_confchg_callback(cpg_handle_t handle,
48 const struct cpg_name *groupName,
49 const struct cpg_address *member_list, size_t member_list_entries,
50 const struct cpg_address *left_list, size_t left_list_entries,
51 const struct cpg_address *joined_list, size_t joined_list_entries);
53 static void _cluster_closedown(void);
55 /* Hash list of nodes in the cluster */
56 static struct dm_hash_table *node_hash;
58 /* For associating lock IDs & resource handles */
59 static struct dm_hash_table *lock_hash;
61 /* Number of active nodes */
63 static unsigned int our_nodeid;
65 static struct local_client *cluster_client;
68 static cpg_handle_t cpg_handle;
69 static SaLckHandleT lck_handle;
71 static struct cpg_name cpg_group_name;
73 /* Openais callback structs */
74 cpg_callbacks_t openais_cpg_callbacks = {
75 .cpg_deliver_fn = openais_cpg_deliver_callback,
76 .cpg_confchg_fn = openais_cpg_confchg_callback,
81 enum {NODE_UNKNOWN, NODE_DOWN, NODE_UP, NODE_CLVMD} state;
87 SaLckResourceHandleT res_handle;
92 /* Set errno to something approximating the right value and return 0 or -1 */
93 static int ais_to_errno(SaAisErrorT err)
99 case SA_AIS_ERR_LIBRARY:
102 case SA_AIS_ERR_VERSION:
105 case SA_AIS_ERR_INIT:
108 case SA_AIS_ERR_TIMEOUT:
111 case SA_AIS_ERR_TRY_AGAIN:
114 case SA_AIS_ERR_INVALID_PARAM:
117 case SA_AIS_ERR_NO_MEMORY:
120 case SA_AIS_ERR_BAD_HANDLE:
123 case SA_AIS_ERR_BUSY:
126 case SA_AIS_ERR_ACCESS:
129 case SA_AIS_ERR_NOT_EXIST:
132 case SA_AIS_ERR_NAME_TOO_LONG:
133 errno = ENAMETOOLONG;
135 case SA_AIS_ERR_EXIST:
138 case SA_AIS_ERR_NO_SPACE:
141 case SA_AIS_ERR_INTERRUPT:
144 case SA_AIS_ERR_NAME_NOT_FOUND:
147 case SA_AIS_ERR_NO_RESOURCES:
150 case SA_AIS_ERR_NOT_SUPPORTED:
153 case SA_AIS_ERR_BAD_OPERATION:
156 case SA_AIS_ERR_FAILED_OPERATION:
159 case SA_AIS_ERR_MESSAGE_ERROR:
162 case SA_AIS_ERR_QUEUE_FULL:
165 case SA_AIS_ERR_QUEUE_NOT_AVAILABLE:
168 case SA_AIS_ERR_BAD_FLAGS:
171 case SA_AIS_ERR_TOO_BIG:
174 case SA_AIS_ERR_NO_SECTIONS:
184 static char *print_openais_csid(const char *csid)
186 static char buf[128];
189 memcpy(&id, csid, sizeof(int));
190 sprintf(buf, "%d", id);
194 static int add_internal_client(int fd, fd_callback_t callback)
196 struct local_client *client;
198 DEBUGLOG("Add_internal_client, fd = %d\n", fd);
200 client = malloc(sizeof(struct local_client));
203 DEBUGLOG("malloc failed\n");
207 memset(client, 0, sizeof(struct local_client));
209 client->type = CLUSTER_INTERNAL;
210 client->callback = callback;
213 /* Set Close-on-exec */
214 fcntl(fd, F_SETFD, 1);
219 static void openais_cpg_deliver_callback (cpg_handle_t handle,
220 const struct cpg_name *groupName,
228 memcpy(&target_nodeid, msg, OPENAIS_CSID_LEN);
230 DEBUGLOG("%u got message from nodeid %d for %d. len %d\n",
231 our_nodeid, nodeid, target_nodeid, msg_len-4);
233 if (nodeid != our_nodeid)
234 if (target_nodeid == our_nodeid || target_nodeid == 0)
235 process_message(cluster_client, (char *)msg+OPENAIS_CSID_LEN,
236 msg_len-OPENAIS_CSID_LEN, (char*)&nodeid);
239 static void openais_cpg_confchg_callback(cpg_handle_t handle,
240 const struct cpg_name *groupName,
241 const struct cpg_address *member_list, size_t member_list_entries,
242 const struct cpg_address *left_list, size_t left_list_entries,
243 const struct cpg_address *joined_list, size_t joined_list_entries)
246 struct node_info *ninfo;
248 DEBUGLOG("confchg callback. %d joined, %d left, %d members\n",
249 joined_list_entries, left_list_entries, member_list_entries);
251 for (i=0; i<joined_list_entries; i++) {
252 ninfo = dm_hash_lookup_binary(node_hash,
253 (char *)&joined_list[i].nodeid,
256 ninfo = malloc(sizeof(struct node_info));
261 ninfo->nodeid = joined_list[i].nodeid;
262 dm_hash_insert_binary(node_hash,
263 (char *)&ninfo->nodeid,
264 OPENAIS_CSID_LEN, ninfo);
267 ninfo->state = NODE_CLVMD;
270 for (i=0; i<left_list_entries; i++) {
271 ninfo = dm_hash_lookup_binary(node_hash,
272 (char *)&left_list[i].nodeid,
275 ninfo->state = NODE_DOWN;
278 for (i=0; i<member_list_entries; i++) {
279 if (member_list[i].nodeid == 0) continue;
280 ninfo = dm_hash_lookup_binary(node_hash,
281 (char *)&member_list[i].nodeid,
284 ninfo = malloc(sizeof(struct node_info));
289 ninfo->nodeid = member_list[i].nodeid;
290 dm_hash_insert_binary(node_hash,
291 (char *)&ninfo->nodeid,
292 OPENAIS_CSID_LEN, ninfo);
295 ninfo->state = NODE_CLVMD;
298 num_nodes = member_list_entries;
301 static int lck_dispatch(struct local_client *client, char *buf, int len,
302 const char *csid, struct local_client **new_client)
305 saLckDispatch(lck_handle, SA_DISPATCH_ONE);
309 static int _init_cluster(void)
312 SaVersionT ver = { 'B', 1, 1 };
315 node_hash = dm_hash_create(100);
316 lock_hash = dm_hash_create(10);
318 err = cpg_initialize(&cpg_handle,
319 &openais_cpg_callbacks);
320 if (err != SA_AIS_OK) {
321 syslog(LOG_ERR, "Cannot initialise OpenAIS CPG service: %d",
323 DEBUGLOG("Cannot initialise OpenAIS CPG service: %d", err);
324 return ais_to_errno(err);
327 err = saLckInitialize(&lck_handle,
330 if (err != SA_AIS_OK) {
331 cpg_initialize(&cpg_handle, &openais_cpg_callbacks);
332 syslog(LOG_ERR, "Cannot initialise OpenAIS lock service: %d",
334 DEBUGLOG("Cannot initialise OpenAIS lock service: %d\n\n", err);
335 return ais_to_errno(err);
338 /* Connect to the clvmd group */
339 strcpy((char *)cpg_group_name.value, "clvmd");
340 cpg_group_name.length = strlen((char *)cpg_group_name.value);
341 err = cpg_join(cpg_handle, &cpg_group_name);
342 if (err != SA_AIS_OK) {
343 cpg_finalize(cpg_handle);
344 saLckFinalize(lck_handle);
345 syslog(LOG_ERR, "Cannot join clvmd process group");
346 DEBUGLOG("Cannot join clvmd process group: %d\n", err);
347 return ais_to_errno(err);
350 err = cpg_local_get(cpg_handle,
352 if (err != SA_AIS_OK) {
353 cpg_finalize(cpg_handle);
354 saLckFinalize(lck_handle);
355 syslog(LOG_ERR, "Cannot get local node id\n");
356 return ais_to_errno(err);
358 DEBUGLOG("Our local node id is %d\n", our_nodeid);
360 saLckSelectionObjectGet(lck_handle, (SaSelectionObjectT *)&select_fd);
361 add_internal_client(select_fd, lck_dispatch);
363 DEBUGLOG("Connected to OpenAIS\n");
368 static void _cluster_closedown(void)
370 DEBUGLOG("cluster_closedown\n");
373 saLckFinalize(lck_handle);
374 cpg_finalize(cpg_handle);
377 static void _get_our_csid(char *csid)
379 memcpy(csid, &our_nodeid, sizeof(int));
382 /* OpenAIS doesn't really have nmode names so we
383 just use the node ID in hex instead */
384 static int _csid_from_name(char *csid, const char *name)
387 struct node_info *ninfo;
389 if (sscanf(name, "%x", &nodeid) == 1) {
390 ninfo = dm_hash_lookup_binary(node_hash, csid, OPENAIS_CSID_LEN);
397 static int _name_from_csid(const char *csid, char *name)
399 struct node_info *ninfo;
401 ninfo = dm_hash_lookup_binary(node_hash, csid, OPENAIS_CSID_LEN);
404 sprintf(name, "UNKNOWN %s", print_openais_csid(csid));
408 sprintf(name, "%x", ninfo->nodeid);
412 static int _get_num_nodes()
414 DEBUGLOG("num_nodes = %d\n", num_nodes);
418 /* Node is now known to be running a clvmd */
419 static void _add_up_node(const char *csid)
421 struct node_info *ninfo;
423 ninfo = dm_hash_lookup_binary(node_hash, csid, OPENAIS_CSID_LEN);
425 DEBUGLOG("openais_add_up_node no node_hash entry for csid %s\n",
426 print_openais_csid(csid));
430 DEBUGLOG("openais_add_up_node %d\n", ninfo->nodeid);
432 ninfo->state = NODE_CLVMD;
437 /* Call a callback for each node, so the caller knows whether it's up or down */
438 static int _cluster_do_node_callback(struct local_client *master_client,
439 void (*callback)(struct local_client *,
440 const char *csid, int node_up))
442 struct dm_hash_node *hn;
443 struct node_info *ninfo;
446 dm_hash_iterate(hn, node_hash)
448 char csid[OPENAIS_CSID_LEN];
450 ninfo = dm_hash_get_data(node_hash, hn);
451 memcpy(csid, dm_hash_get_key(node_hash, hn), OPENAIS_CSID_LEN);
453 DEBUGLOG("down_callback. node %d, state = %d\n", ninfo->nodeid,
456 if (ninfo->state != NODE_DOWN)
457 callback(master_client, csid, ninfo->state == NODE_CLVMD);
458 if (ninfo->state != NODE_CLVMD)
465 static int _lock_resource(char *resource, int mode, int flags, int *lockid)
467 struct lock_info *linfo;
468 SaLckResourceHandleT res_handle;
470 SaLckLockIdT lock_id;
471 SaLckLockStatusT lockStatus;
473 /* This needs to be converted from DLM/LVM2 value for OpenAIS LCK */
474 if (flags & LCK_NONBLOCK) flags = SA_LCK_LOCK_NO_QUEUE;
476 linfo = malloc(sizeof(struct lock_info));
480 DEBUGLOG("lock_resource '%s', flags=%d, mode=%d\n", resource, flags, mode);
482 linfo->lock_name.length = strlen(resource)+1;
483 strcpy((char *)linfo->lock_name.value, resource);
485 err = saLckResourceOpen(lck_handle, &linfo->lock_name,
486 SA_LCK_RESOURCE_CREATE, TIMEOUT, &res_handle);
487 if (err != SA_AIS_OK)
489 DEBUGLOG("ResourceOpen returned %d\n", err);
491 return ais_to_errno(err);
494 err = saLckResourceLock(
502 if (err != SA_AIS_OK && lockStatus != SA_LCK_LOCK_GRANTED)
505 saLckResourceClose(res_handle);
506 return ais_to_errno(err);
509 /* Wait for it to complete */
511 DEBUGLOG("lock_resource returning %d, lock_id=%llx\n", err,
514 linfo->lock_id = lock_id;
515 linfo->res_handle = res_handle;
517 dm_hash_insert(lock_hash, resource, linfo);
519 return ais_to_errno(err);
523 static int _unlock_resource(char *resource, int lockid)
526 struct lock_info *linfo;
528 DEBUGLOG("unlock_resource %s\n", resource);
529 linfo = dm_hash_lookup(lock_hash, resource);
533 DEBUGLOG("unlock_resource: lockid: %llx\n", linfo->lock_id);
534 err = saLckResourceUnlock(linfo->lock_id, SA_TIME_END);
535 if (err != SA_AIS_OK)
537 DEBUGLOG("Unlock returned %d\n", err);
538 return ais_to_errno(err);
541 /* Release the resource */
542 dm_hash_remove(lock_hash, resource);
543 saLckResourceClose(linfo->res_handle);
546 return ais_to_errno(err);
549 static int _sync_lock(const char *resource, int mode, int flags, int *lockid)
552 char lock1[strlen(resource)+3];
553 char lock2[strlen(resource)+3];
555 snprintf(lock1, sizeof(lock1), "%s-1", resource);
556 snprintf(lock2, sizeof(lock2), "%s-2", resource);
561 status = _lock_resource(lock1, SA_LCK_EX_LOCK_MODE, flags, lockid);
565 /* If we can't get this lock too then bail out */
566 status = _lock_resource(lock2, SA_LCK_EX_LOCK_MODE, LCK_NONBLOCK,
568 if (status == SA_LCK_LOCK_NOT_QUEUED)
570 _unlock_resource(lock1, *lockid);
578 status = _lock_resource(lock1, SA_LCK_PR_LOCK_MODE, flags, lockid);
581 _unlock_resource(lock2, *lockid);
585 status = _lock_resource(lock2, SA_LCK_EX_LOCK_MODE, flags, lockid);
588 _unlock_resource(lock1, *lockid);
601 static int _sync_unlock(const char *resource, int lockid)
604 char lock1[strlen(resource)+3];
605 char lock2[strlen(resource)+3];
607 snprintf(lock1, sizeof(lock1), "%s-1", resource);
608 snprintf(lock2, sizeof(lock2), "%s-2", resource);
610 _unlock_resource(lock1, lockid);
611 _unlock_resource(lock2, lockid);
616 /* We are always quorate ! */
617 static int _is_quorate()
622 static int _get_main_cluster_fd(void)
626 cpg_fd_get(cpg_handle, &select_fd);
630 static int _cluster_fd_callback(struct local_client *fd, char *buf, int len,
632 struct local_client **new_client)
636 cpg_dispatch(cpg_handle, SA_DISPATCH_ONE);
640 static int _cluster_send_message(const void *buf, int msglen, const char *csid,
648 memcpy(&target_node, csid, OPENAIS_CSID_LEN);
652 iov[0].iov_base = &target_node;
653 iov[0].iov_len = sizeof(int);
654 iov[1].iov_base = (char *)buf;
655 iov[1].iov_len = msglen;
657 err = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2);
658 return ais_to_errno(err);
661 /* We don't have a cluster name to report here */
662 static int _get_cluster_name(char *buf, int buflen)
664 strncpy(buf, "OpenAIS", buflen);
668 static struct cluster_ops _cluster_openais_ops = {
669 .cluster_init_completed = NULL,
670 .cluster_send_message = _cluster_send_message,
671 .name_from_csid = _name_from_csid,
672 .csid_from_name = _csid_from_name,
673 .get_num_nodes = _get_num_nodes,
674 .cluster_fd_callback = _cluster_fd_callback,
675 .get_main_cluster_fd = _get_main_cluster_fd,
676 .cluster_do_node_callback = _cluster_do_node_callback,
677 .is_quorate = _is_quorate,
678 .get_our_csid = _get_our_csid,
679 .add_up_node = _add_up_node,
680 .reread_config = NULL,
681 .cluster_closedown = _cluster_closedown,
682 .get_cluster_name = _get_cluster_name,
683 .sync_lock = _sync_lock,
684 .sync_unlock = _sync_unlock,
687 struct cluster_ops *init_openais_cluster(void)
689 if (!_init_cluster())
690 return &_cluster_openais_ops;