src/qemu/qemu_migration.c

   1 /*
   2  * qemu_migration.c: QEMU migration handling
   3  *
   4  * Copyright (C) 2006-2014 Red Hat, Inc.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library.  If not, see
  18  * <http://www.gnu.org/licenses/>.
  19  *
  20  */
  21
  22 #include <config.h>
  23
  24 #include <netdb.h>
  25 #include <sys/socket.h>
  26 #include <sys/time.h>
  27 #ifdef WITH_GNUTLS
  28 # include <gnutls/gnutls.h>
  29 # include <gnutls/x509.h>
  30 #endif
  31 #include <fcntl.h>
  32 #include <poll.h>
  33
  34 #include "qemu_migration.h"
  35 #include "qemu_monitor.h"
  36 #include "qemu_domain.h"
  37 #include "qemu_process.h"
  38 #include "qemu_capabilities.h"
  39 #include "qemu_command.h"
  40 #include "qemu_cgroup.h"
  41 #include "qemu_hotplug.h"
  42
  43 #include "domain_audit.h"
  44 #include "virlog.h"
  45 #include "virerror.h"
  46 #include "viralloc.h"
  47 #include "virfile.h"
  48 #include "datatypes.h"
  49 #include "fdstream.h"
  50 #include "viruuid.h"
  51 #include "virtime.h"
  52 #include "locking/domain_lock.h"
  53 #include "rpc/virnetsocket.h"
  54 #include "virstoragefile.h"
  55 #include "viruri.h"
  56 #include "virhook.h"
  57 #include "virstring.h"
  58 #include "virtypedparam.h"
  59
  60 #define VIR_FROM_THIS VIR_FROM_QEMU
  61
  62 VIR_LOG_INIT("qemu.qemu_migration");
  63
  64 VIR_ENUM_IMPL(qemuMigrationJobPhase, QEMU_MIGRATION_PHASE_LAST,
  65               "none",
  66               "perform2",
  67               "begin3",
  68               "perform3",
  69               "perform3_done",
  70               "confirm3_cancelled",
  71               "confirm3",
  72               "prepare",
  73               "finish2",
  74               "finish3",
  75 );
  76
  77 enum qemuMigrationCookieFlags {
  78     QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS,
  79     QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE,
  80     QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT,
  81     QEMU_MIGRATION_COOKIE_FLAG_NETWORK,
  82     QEMU_MIGRATION_COOKIE_FLAG_NBD,
  83
  84     QEMU_MIGRATION_COOKIE_FLAG_LAST
  85 };
  86
  87 VIR_ENUM_DECL(qemuMigrationCookieFlag);
  88 VIR_ENUM_IMPL(qemuMigrationCookieFlag,
  89               QEMU_MIGRATION_COOKIE_FLAG_LAST,
  90               "graphics",
  91               "lockstate",
  92               "persistent",
  93               "network",
  94               "nbd");
  95
  96 enum qemuMigrationCookieFeatures {
  97     QEMU_MIGRATION_COOKIE_GRAPHICS  = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS),
  98     QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE),
  99     QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT),
 100     QEMU_MIGRATION_COOKIE_NETWORK = (1 << QEMU_MIGRATION_COOKIE_FLAG_NETWORK),
 101     QEMU_MIGRATION_COOKIE_NBD = (1 << QEMU_MIGRATION_COOKIE_FLAG_NBD),
 102 };
 103
 104 typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics;
 105 typedef qemuMigrationCookieGraphics *qemuMigrationCookieGraphicsPtr;
 106 struct _qemuMigrationCookieGraphics {
 107     int type;
 108     int port;
 109     int tlsPort;
 110     char *listen;
 111     char *tlsSubject;
 112 };
 113
 114 typedef struct _qemuMigrationCookieNetData qemuMigrationCookieNetData;
 115 typedef qemuMigrationCookieNetData *qemuMigrationCookieNetDataPtr;
 116 struct _qemuMigrationCookieNetData {
 117     int vporttype; /* enum virNetDevVPortProfile */
 118
 119     /*
 120      * Array of pointers to saved data. Each VIF will have it's own
 121      * data to transfer.
 122      */
 123     char *portdata;
 124 };
 125
 126 typedef struct _qemuMigrationCookieNetwork qemuMigrationCookieNetwork;
 127 typedef qemuMigrationCookieNetwork *qemuMigrationCookieNetworkPtr;
 128 struct _qemuMigrationCookieNetwork {
 129     /* How many virtual NICs are we saving data for? */
 130     int nnets;
 131
 132     qemuMigrationCookieNetDataPtr net;
 133 };
 134
 135 typedef struct _qemuMigrationCookieNBD qemuMigrationCookieNBD;
 136 typedef qemuMigrationCookieNBD *qemuMigrationCookieNBDPtr;
 137 struct _qemuMigrationCookieNBD {
 138     int port; /* on which port does NBD server listen for incoming data */
 139 };
 140
 141 typedef struct _qemuMigrationCookie qemuMigrationCookie;
 142 typedef qemuMigrationCookie *qemuMigrationCookiePtr;
 143 struct _qemuMigrationCookie {
 144     unsigned int flags;
 145     unsigned int flagsMandatory;
 146
 147     /* Host properties */
 148     unsigned char localHostuuid[VIR_UUID_BUFLEN];
 149     unsigned char remoteHostuuid[VIR_UUID_BUFLEN];
 150     char *localHostname;
 151     char *remoteHostname;
 152
 153     /* Guest properties */
 154     unsigned char uuid[VIR_UUID_BUFLEN];
 155     char *name;
 156
 157     /* If (flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) */
 158     char *lockState;
 159     char *lockDriver;
 160
 161     /* If (flags & QEMU_MIGRATION_COOKIE_GRAPHICS) */
 162     qemuMigrationCookieGraphicsPtr graphics;
 163
 164     /* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */
 165     virDomainDefPtr persistent;
 166
 167     /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */
 168     qemuMigrationCookieNetworkPtr network;
 169
 170     /* If (flags & QEMU_MIGRATION_COOKIE_NBD) */
 171     qemuMigrationCookieNBDPtr nbd;
 172 };
 173
 174 static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap)
 175 {
 176     if (!grap)
 177         return;
 178     VIR_FREE(grap->listen);
 179     VIR_FREE(grap->tlsSubject);
 180     VIR_FREE(grap);
 181 }
 182
 183
 184 static void
 185 qemuMigrationCookieNetworkFree(qemuMigrationCookieNetworkPtr network)
 186 {
 187     size_t i;
 188
 189     if (!network)
 190         return;
 191
 192     if (network->net) {
 193         for (i = 0; i < network->nnets; i++)
 194             VIR_FREE(network->net[i].portdata);
 195     }
 196     VIR_FREE(network->net);
 197     VIR_FREE(network);
 198 }
 199
 200
 201 static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig)
 202 {
 203     if (!mig)
 204         return;
 205
 206     qemuMigrationCookieGraphicsFree(mig->graphics);
 207     qemuMigrationCookieNetworkFree(mig->network);
 208
 209     VIR_FREE(mig->localHostname);
 210     VIR_FREE(mig->remoteHostname);
 211     VIR_FREE(mig->name);
 212     VIR_FREE(mig->lockState);
 213     VIR_FREE(mig->lockDriver);
 214     VIR_FREE(mig->nbd);
 215     VIR_FREE(mig);
 216 }
 217
 218
 219 #ifdef WITH_GNUTLS
 220 static char *
 221 qemuDomainExtractTLSSubject(const char *certdir)
 222 {
 223     char *certfile = NULL;
 224     char *subject = NULL;
 225     char *pemdata = NULL;
 226     gnutls_datum_t pemdatum;
 227     gnutls_x509_crt_t cert;
 228     int ret;
 229     size_t subjectlen;
 230
 231     if (virAsprintf(&certfile, "%s/server-cert.pem", certdir) < 0)
 232         goto error;
 233
 234     if (virFileReadAll(certfile, 8192, &pemdata) < 0) {
 235         virReportError(VIR_ERR_INTERNAL_ERROR,
 236                        _("unable to read server cert %s"), certfile);
 237         goto error;
 238     }
 239
 240     ret = gnutls_x509_crt_init(&cert);
 241     if (ret < 0) {
 242         virReportError(VIR_ERR_INTERNAL_ERROR,
 243                        _("cannot initialize cert object: %s"),
 244                        gnutls_strerror(ret));
 245         goto error;
 246     }
 247
 248     pemdatum.data = (unsigned char *)pemdata;
 249     pemdatum.size = strlen(pemdata);
 250
 251     ret = gnutls_x509_crt_import(cert, &pemdatum, GNUTLS_X509_FMT_PEM);
 252     if (ret < 0) {
 253         virReportError(VIR_ERR_INTERNAL_ERROR,
 254                        _("cannot load cert data from %s: %s"),
 255                        certfile, gnutls_strerror(ret));
 256         goto error;
 257     }
 258
 259     subjectlen = 1024;
 260     if (VIR_ALLOC_N(subject, subjectlen+1) < 0)
 261         goto error;
 262
 263     gnutls_x509_crt_get_dn(cert, subject, &subjectlen);
 264     subject[subjectlen] = '\0';
 265
 266     VIR_FREE(certfile);
 267     VIR_FREE(pemdata);
 268
 269     return subject;
 270
 271  error:
 272     VIR_FREE(certfile);
 273     VIR_FREE(pemdata);
 274     return NULL;
 275 }
 276 #endif
 277
 278 static qemuMigrationCookieGraphicsPtr
 279 qemuMigrationCookieGraphicsAlloc(virQEMUDriverPtr driver,
 280                                  virDomainGraphicsDefPtr def)
 281 {
 282     qemuMigrationCookieGraphicsPtr mig = NULL;
 283     const char *listenAddr;
 284     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
 285
 286     if (VIR_ALLOC(mig) < 0)
 287         goto error;
 288
 289     mig->type = def->type;
 290     if (mig->type == VIR_DOMAIN_GRAPHICS_TYPE_VNC) {
 291         mig->port = def->data.vnc.port;
 292         listenAddr = virDomainGraphicsListenGetAddress(def, 0);
 293         if (!listenAddr)
 294             listenAddr = cfg->vncListen;
 295
 296 #ifdef WITH_GNUTLS
 297         if (cfg->vncTLS &&
 298             !(mig->tlsSubject = qemuDomainExtractTLSSubject(cfg->vncTLSx509certdir)))
 299             goto error;
 300 #endif
 301     } else {
 302         mig->port = def->data.spice.port;
 303         if (cfg->spiceTLS)
 304             mig->tlsPort = def->data.spice.tlsPort;
 305         else
 306             mig->tlsPort = -1;
 307         listenAddr = virDomainGraphicsListenGetAddress(def, 0);
 308         if (!listenAddr)
 309             listenAddr = cfg->spiceListen;
 310
 311 #ifdef WITH_GNUTLS
 312         if (cfg->spiceTLS &&
 313             !(mig->tlsSubject = qemuDomainExtractTLSSubject(cfg->spiceTLSx509certdir)))
 314             goto error;
 315 #endif
 316     }
 317     if (VIR_STRDUP(mig->listen, listenAddr) < 0)
 318         goto error;
 319
 320     virObjectUnref(cfg);
 321     return mig;
 322
 323  error:
 324     qemuMigrationCookieGraphicsFree(mig);
 325     virObjectUnref(cfg);
 326     return NULL;
 327 }
 328
 329
 330 static qemuMigrationCookieNetworkPtr
 331 qemuMigrationCookieNetworkAlloc(virQEMUDriverPtr driver ATTRIBUTE_UNUSED,
 332                                 virDomainDefPtr def)
 333 {
 334     qemuMigrationCookieNetworkPtr mig;
 335     size_t i;
 336
 337     if (VIR_ALLOC(mig) < 0)
 338         goto error;
 339
 340     mig->nnets = def->nnets;
 341
 342     if (VIR_ALLOC_N(mig->net, def->nnets) <0)
 343         goto error;
 344
 345     for (i = 0; i < def->nnets; i++) {
 346         virDomainNetDefPtr netptr;
 347         virNetDevVPortProfilePtr vport;
 348
 349         netptr = def->nets[i];
 350         vport = virDomainNetGetActualVirtPortProfile(netptr);
 351
 352         if (vport) {
 353             mig->net[i].vporttype = vport->virtPortType;
 354
 355             switch (vport->virtPortType) {
 356             case VIR_NETDEV_VPORT_PROFILE_NONE:
 357             case VIR_NETDEV_VPORT_PROFILE_8021QBG:
 358             case VIR_NETDEV_VPORT_PROFILE_8021QBH:
 359                break;
 360             case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH:
 361                 if (virNetDevOpenvswitchGetMigrateData(&mig->net[i].portdata,
 362                                                        netptr->ifname) != 0) {
 363                         virReportSystemError(VIR_ERR_INTERNAL_ERROR,
 364                                              _("Unable to run command to get OVS port data for "
 365                                              "interface %s"), netptr->ifname);
 366                         goto error;
 367                 }
 368                 break;
 369             default:
 370                 break;
 371             }
 372         }
 373     }
 374     return mig;
 375
 376  error:
 377     qemuMigrationCookieNetworkFree(mig);
 378     return NULL;
 379 }
 380
 381 static qemuMigrationCookiePtr
 382 qemuMigrationCookieNew(virDomainObjPtr dom)
 383 {
 384     qemuDomainObjPrivatePtr priv = dom->privateData;
 385     qemuMigrationCookiePtr mig = NULL;
 386     const char *name;
 387
 388     if (VIR_ALLOC(mig) < 0)
 389         goto error;
 390
 391     if (priv->origname)
 392         name = priv->origname;
 393     else
 394         name = dom->def->name;
 395     if (VIR_STRDUP(mig->name, name) < 0)
 396         goto error;
 397     memcpy(mig->uuid, dom->def->uuid, VIR_UUID_BUFLEN);
 398
 399     if (!(mig->localHostname = virGetHostname()))
 400         goto error;
 401     if (virGetHostUUID(mig->localHostuuid) < 0) {
 402         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 403                        _("Unable to obtain host UUID"));
 404         goto error;
 405     }
 406
 407     return mig;
 408
 409  error:
 410     qemuMigrationCookieFree(mig);
 411     return NULL;
 412 }
 413
 414
 415 static int
 416 qemuMigrationCookieAddGraphics(qemuMigrationCookiePtr mig,
 417                                virQEMUDriverPtr driver,
 418                                virDomainObjPtr dom)
 419 {
 420     size_t i = 0;
 421
 422     if (mig->flags & QEMU_MIGRATION_COOKIE_GRAPHICS) {
 423         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 424                        _("Migration graphics data already present"));
 425         return -1;
 426     }
 427
 428     for (i = 0; i < dom->def->ngraphics; i++) {
 429        if (dom->def->graphics[i]->type == VIR_DOMAIN_GRAPHICS_TYPE_SPICE) {
 430            if (!(mig->graphics =
 431                  qemuMigrationCookieGraphicsAlloc(driver, dom->def->graphics[i])))
 432                return -1;
 433            mig->flags |= QEMU_MIGRATION_COOKIE_GRAPHICS;
 434            break;
 435        }
 436     }
 437
 438     return 0;
 439 }
 440
 441
 442 static int
 443 qemuMigrationCookieAddLockstate(qemuMigrationCookiePtr mig,
 444                                 virQEMUDriverPtr driver,
 445                                 virDomainObjPtr dom)
 446 {
 447     qemuDomainObjPrivatePtr priv = dom->privateData;
 448
 449     if (mig->flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) {
 450         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 451                        _("Migration lockstate data already present"));
 452         return -1;
 453     }
 454
 455     if (virDomainObjGetState(dom, NULL) == VIR_DOMAIN_PAUSED) {
 456         if (VIR_STRDUP(mig->lockState, priv->lockState) < 0)
 457             return -1;
 458     } else {
 459         if (virDomainLockProcessInquire(driver->lockManager, dom, &mig->lockState) < 0)
 460             return -1;
 461     }
 462
 463     if (VIR_STRDUP(mig->lockDriver, virLockManagerPluginGetName(driver->lockManager)) < 0) {
 464         VIR_FREE(mig->lockState);
 465         return -1;
 466     }
 467
 468     mig->flags |= QEMU_MIGRATION_COOKIE_LOCKSTATE;
 469     mig->flagsMandatory |= QEMU_MIGRATION_COOKIE_LOCKSTATE;
 470
 471     return 0;
 472 }
 473
 474
 475 static int
 476 qemuMigrationCookieAddPersistent(qemuMigrationCookiePtr mig,
 477                                  virDomainObjPtr dom)
 478 {
 479     if (mig->flags & QEMU_MIGRATION_COOKIE_PERSISTENT) {
 480         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 481                        _("Migration persistent data already present"));
 482         return -1;
 483     }
 484
 485     if (!dom->newDef)
 486         return 0;
 487
 488     mig->persistent = dom->newDef;
 489     mig->flags |= QEMU_MIGRATION_COOKIE_PERSISTENT;
 490     mig->flagsMandatory |= QEMU_MIGRATION_COOKIE_PERSISTENT;
 491     return 0;
 492 }
 493
 494
 495 static int
 496 qemuMigrationCookieAddNetwork(qemuMigrationCookiePtr mig,
 497                               virQEMUDriverPtr driver,
 498                               virDomainObjPtr dom)
 499 {
 500     if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) {
 501         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 502                        _("Network migration data already present"));
 503         return -1;
 504     }
 505
 506     if (dom->def->nnets > 0) {
 507         mig->network = qemuMigrationCookieNetworkAlloc(driver, dom->def);
 508         if (!mig->network)
 509             return -1;
 510         mig->flags |= QEMU_MIGRATION_COOKIE_NETWORK;
 511     }
 512
 513     return 0;
 514 }
 515
 516
 517 static int
 518 qemuMigrationCookieAddNBD(qemuMigrationCookiePtr mig,
 519                           virQEMUDriverPtr driver ATTRIBUTE_UNUSED,
 520                           virDomainObjPtr vm)
 521 {
 522     qemuDomainObjPrivatePtr priv = vm->privateData;
 523
 524     /* It is not a bug if there already is a NBD data */
 525     if (!mig->nbd &&
 526         VIR_ALLOC(mig->nbd) < 0)
 527         return -1;
 528
 529     mig->nbd->port = priv->nbdPort;
 530     mig->flags |= QEMU_MIGRATION_COOKIE_NBD;
 531
 532     return 0;
 533 }
 534
 535
 536 static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf,
 537                                                  qemuMigrationCookieGraphicsPtr grap)
 538 {
 539     virBufferAsprintf(buf, "<graphics type='%s' port='%d' listen='%s'",
 540                       virDomainGraphicsTypeToString(grap->type),
 541                       grap->port, grap->listen);
 542     if (grap->type == VIR_DOMAIN_GRAPHICS_TYPE_SPICE)
 543         virBufferAsprintf(buf, " tlsPort='%d'", grap->tlsPort);
 544     if (grap->tlsSubject) {
 545         virBufferAddLit(buf, ">\n");
 546         virBufferAdjustIndent(buf, 2);
 547         virBufferEscapeString(buf, "<cert info='subject' value='%s'/>\n", grap->tlsSubject);
 548         virBufferAdjustIndent(buf, -2);
 549         virBufferAddLit(buf, "</graphics>\n");
 550     } else {
 551         virBufferAddLit(buf, "/>\n");
 552     }
 553 }
 554
 555
 556 static void
 557 qemuMigrationCookieNetworkXMLFormat(virBufferPtr buf,
 558                                     qemuMigrationCookieNetworkPtr optr)
 559 {
 560     size_t i;
 561     bool empty = true;
 562
 563     for (i = 0; i < optr->nnets; i++) {
 564         /* If optr->net[i].vporttype is not set, there is nothing to transfer */
 565         if (optr->net[i].vporttype != VIR_NETDEV_VPORT_PROFILE_NONE) {
 566             if (empty) {
 567                 virBufferAddLit(buf, "<network>\n");
 568                 virBufferAdjustIndent(buf, 2);
 569                 empty = false;
 570             }
 571             virBufferAsprintf(buf, "<interface index='%zu' vporttype='%s'",
 572                               i, virNetDevVPortTypeToString(optr->net[i].vporttype));
 573             if (optr->net[i].portdata) {
 574                 virBufferAddLit(buf, ">\n");
 575                 virBufferAdjustIndent(buf, 2);
 576                 virBufferEscapeString(buf, "<portdata>%s</portdata>\n",
 577                                       optr->net[i].portdata);
 578                 virBufferAdjustIndent(buf, -2);
 579                 virBufferAddLit(buf, "</interface>\n");
 580             } else {
 581                 virBufferAddLit(buf, "/>\n");
 582             }
 583         }
 584     }
 585     if (!empty) {
 586         virBufferAdjustIndent(buf, -2);
 587         virBufferAddLit(buf, "</network>\n");
 588     }
 589 }
 590
 591
 592 static int
 593 qemuMigrationCookieXMLFormat(virQEMUDriverPtr driver,
 594                              virBufferPtr buf,
 595                              qemuMigrationCookiePtr mig)
 596 {
 597     char uuidstr[VIR_UUID_STRING_BUFLEN];
 598     char hostuuidstr[VIR_UUID_STRING_BUFLEN];
 599     size_t i;
 600
 601     virUUIDFormat(mig->uuid, uuidstr);
 602     virUUIDFormat(mig->localHostuuid, hostuuidstr);
 603
 604     virBufferAddLit(buf, "<qemu-migration>\n");
 605     virBufferAdjustIndent(buf, 2);
 606     virBufferEscapeString(buf, "<name>%s</name>\n", mig->name);
 607     virBufferAsprintf(buf, "<uuid>%s</uuid>\n", uuidstr);
 608     virBufferEscapeString(buf, "<hostname>%s</hostname>\n", mig->localHostname);
 609     virBufferAsprintf(buf, "<hostuuid>%s</hostuuid>\n", hostuuidstr);
 610
 611     for (i = 0; i < QEMU_MIGRATION_COOKIE_FLAG_LAST; i++) {
 612         if (mig->flagsMandatory & (1 << i))
 613             virBufferAsprintf(buf, "<feature name='%s'/>\n",
 614                               qemuMigrationCookieFlagTypeToString(i));
 615     }
 616
 617     if ((mig->flags & QEMU_MIGRATION_COOKIE_GRAPHICS) &&
 618         mig->graphics)
 619         qemuMigrationCookieGraphicsXMLFormat(buf, mig->graphics);
 620
 621     if ((mig->flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) &&
 622         mig->lockState) {
 623         virBufferAsprintf(buf, "<lockstate driver='%s'>\n",
 624                           mig->lockDriver);
 625         virBufferAdjustIndent(buf, 2);
 626         virBufferAsprintf(buf, "<leases>%s</leases>\n",
 627                           mig->lockState);
 628         virBufferAdjustIndent(buf, -2);
 629         virBufferAddLit(buf, "</lockstate>\n");
 630     }
 631
 632     if ((mig->flags & QEMU_MIGRATION_COOKIE_PERSISTENT) &&
 633         mig->persistent) {
 634         if (qemuDomainDefFormatBuf(driver,
 635                                    mig->persistent,
 636                                    VIR_DOMAIN_XML_INACTIVE |
 637                                    VIR_DOMAIN_XML_SECURE |
 638                                    VIR_DOMAIN_XML_MIGRATABLE,
 639                                    buf) < 0)
 640             return -1;
 641     }
 642
 643     if ((mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) && mig->network)
 644         qemuMigrationCookieNetworkXMLFormat(buf, mig->network);
 645
 646     if ((mig->flags & QEMU_MIGRATION_COOKIE_NBD) && mig->nbd) {
 647         virBufferAddLit(buf, "<nbd");
 648         if (mig->nbd->port)
 649             virBufferAsprintf(buf, " port='%d'", mig->nbd->port);
 650         virBufferAddLit(buf, "/>\n");
 651     }
 652
 653     virBufferAdjustIndent(buf, -2);
 654     virBufferAddLit(buf, "</qemu-migration>\n");
 655     return 0;
 656 }
 657
 658
 659 static char *qemuMigrationCookieXMLFormatStr(virQEMUDriverPtr driver,
 660                                              qemuMigrationCookiePtr mig)
 661 {
 662     virBuffer buf = VIR_BUFFER_INITIALIZER;
 663
 664     if (qemuMigrationCookieXMLFormat(driver, &buf, mig) < 0) {
 665         virBufferFreeAndReset(&buf);
 666         return NULL;
 667     }
 668
 669     if (virBufferError(&buf)) {
 670         virReportOOMError();
 671         virBufferFreeAndReset(&buf);
 672         return NULL;
 673     }
 674
 675     return virBufferContentAndReset(&buf);
 676 }
 677
 678
 679 static qemuMigrationCookieGraphicsPtr
 680 qemuMigrationCookieGraphicsXMLParse(xmlXPathContextPtr ctxt)
 681 {
 682     qemuMigrationCookieGraphicsPtr grap;
 683     char *tmp;
 684
 685     if (VIR_ALLOC(grap) < 0)
 686         goto error;
 687
 688     if (!(tmp = virXPathString("string(./graphics/@type)", ctxt))) {
 689         virReportError(VIR_ERR_INTERNAL_ERROR,
 690                        "%s", _("missing type attribute in migration data"));
 691         goto error;
 692     }
 693     if ((grap->type = virDomainGraphicsTypeFromString(tmp)) < 0) {
 694         virReportError(VIR_ERR_INTERNAL_ERROR,
 695                        _("unknown graphics type %s"), tmp);
 696         VIR_FREE(tmp);
 697         goto error;
 698     }
 699     VIR_FREE(tmp);
 700     if (virXPathInt("string(./graphics/@port)", ctxt, &grap->port) < 0) {
 701         virReportError(VIR_ERR_INTERNAL_ERROR,
 702                        "%s", _("missing port attribute in migration data"));
 703         goto error;
 704     }
 705     if (grap->type == VIR_DOMAIN_GRAPHICS_TYPE_SPICE) {
 706         if (virXPathInt("string(./graphics/@tlsPort)", ctxt, &grap->tlsPort) < 0) {
 707             virReportError(VIR_ERR_INTERNAL_ERROR,
 708                            "%s", _("missing tlsPort attribute in migration data"));
 709             goto error;
 710         }
 711     }
 712     if (!(grap->listen = virXPathString("string(./graphics/@listen)", ctxt))) {
 713         virReportError(VIR_ERR_INTERNAL_ERROR,
 714                        "%s", _("missing listen attribute in migration data"));
 715         goto error;
 716     }
 717     /* Optional */
 718     grap->tlsSubject = virXPathString("string(./graphics/cert[@info='subject']/@value)", ctxt);
 719
 720     return grap;
 721
 722  error:
 723     qemuMigrationCookieGraphicsFree(grap);
 724     return NULL;
 725 }
 726
 727
 728 static qemuMigrationCookieNetworkPtr
 729 qemuMigrationCookieNetworkXMLParse(xmlXPathContextPtr ctxt)
 730 {
 731     qemuMigrationCookieNetworkPtr optr;
 732     size_t i;
 733     int n;
 734     xmlNodePtr *interfaces = NULL;
 735     char *vporttype;
 736     xmlNodePtr save_ctxt = ctxt->node;
 737
 738     if (VIR_ALLOC(optr) < 0)
 739         goto error;
 740
 741     if ((n = virXPathNodeSet("./network/interface", ctxt, &interfaces)) < 0) {
 742         virReportError(VIR_ERR_INTERNAL_ERROR,
 743                        "%s", _("missing interface information"));
 744         goto error;
 745     }
 746
 747     optr->nnets = n;
 748     if (VIR_ALLOC_N(optr->net, optr->nnets) < 0)
 749         goto error;
 750
 751     for (i = 0; i < n; i++) {
 752         /* portdata is optional, and may not exist */
 753         ctxt->node = interfaces[i];
 754         optr->net[i].portdata = virXPathString("string(./portdata[1])", ctxt);
 755
 756         if (!(vporttype = virXMLPropString(interfaces[i], "vporttype"))) {
 757             virReportError(VIR_ERR_INTERNAL_ERROR,
 758                            "%s", _("missing vporttype attribute in migration data"));
 759             goto error;
 760         }
 761         optr->net[i].vporttype = virNetDevVPortTypeFromString(vporttype);
 762     }
 763
 764     VIR_FREE(interfaces);
 765
 766  cleanup:
 767     ctxt->node = save_ctxt;
 768     return optr;
 769
 770  error:
 771     VIR_FREE(interfaces);
 772     qemuMigrationCookieNetworkFree(optr);
 773     optr = NULL;
 774     goto cleanup;
 775 }
 776
 777
 778 static int
 779 qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig,
 780                             virQEMUDriverPtr driver,
 781                             xmlDocPtr doc,
 782                             xmlXPathContextPtr ctxt,
 783                             unsigned int flags)
 784 {
 785     char uuidstr[VIR_UUID_STRING_BUFLEN];
 786     char *tmp = NULL;
 787     xmlNodePtr *nodes = NULL;
 788     size_t i;
 789     int n;
 790     virCapsPtr caps = NULL;
 791
 792     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
 793         goto error;
 794
 795     /* We don't store the uuid, name, hostname, or hostuuid
 796      * values. We just compare them to local data to do some
 797      * sanity checking on migration operation
 798      */
 799
 800     /* Extract domain name */
 801     if (!(tmp = virXPathString("string(./name[1])", ctxt))) {
 802         virReportError(VIR_ERR_INTERNAL_ERROR,
 803                        "%s", _("missing name element in migration data"));
 804         goto error;
 805     }
 806     if (STRNEQ(tmp, mig->name)) {
 807         virReportError(VIR_ERR_INTERNAL_ERROR,
 808                        _("Incoming cookie data had unexpected name %s vs %s"),
 809                        tmp, mig->name);
 810         goto error;
 811     }
 812     VIR_FREE(tmp);
 813
 814     /* Extract domain uuid */
 815     tmp = virXPathString("string(./uuid[1])", ctxt);
 816     if (!tmp) {
 817         virReportError(VIR_ERR_INTERNAL_ERROR,
 818                        "%s", _("missing uuid element in migration data"));
 819         goto error;
 820     }
 821     virUUIDFormat(mig->uuid, uuidstr);
 822     if (STRNEQ(tmp, uuidstr)) {
 823         virReportError(VIR_ERR_INTERNAL_ERROR,
 824                        _("Incoming cookie data had unexpected UUID %s vs %s"),
 825                        tmp, uuidstr);
 826     }
 827     VIR_FREE(tmp);
 828
 829     /* Check & forbid "localhost" migration */
 830     if (!(mig->remoteHostname = virXPathString("string(./hostname[1])", ctxt))) {
 831         virReportError(VIR_ERR_INTERNAL_ERROR,
 832                        "%s", _("missing hostname element in migration data"));
 833         goto error;
 834     }
 835     if (STREQ(mig->remoteHostname, mig->localHostname)) {
 836         virReportError(VIR_ERR_INTERNAL_ERROR,
 837                        _("Attempt to migrate guest to the same host %s"),
 838                        mig->remoteHostname);
 839         goto error;
 840     }
 841
 842     if (!(tmp = virXPathString("string(./hostuuid[1])", ctxt))) {
 843         virReportError(VIR_ERR_INTERNAL_ERROR,
 844                        "%s", _("missing hostuuid element in migration data"));
 845         goto error;
 846     }
 847     if (virUUIDParse(tmp, mig->remoteHostuuid) < 0) {
 848         virReportError(VIR_ERR_INTERNAL_ERROR,
 849                        "%s", _("malformed hostuuid element in migration data"));
 850         goto error;
 851     }
 852     if (memcmp(mig->remoteHostuuid, mig->localHostuuid, VIR_UUID_BUFLEN) == 0) {
 853         virReportError(VIR_ERR_INTERNAL_ERROR,
 854                        _("Attempt to migrate guest to the same host %s"),
 855                        tmp);
 856         goto error;
 857     }
 858     VIR_FREE(tmp);
 859
 860     /* Check to ensure all mandatory features from XML are also
 861      * present in 'flags' */
 862     if ((n = virXPathNodeSet("./feature", ctxt, &nodes)) < 0)
 863         goto error;
 864
 865     for (i = 0; i < n; i++) {
 866         int val;
 867         char *str = virXMLPropString(nodes[i], "name");
 868         if (!str) {
 869             virReportError(VIR_ERR_INTERNAL_ERROR,
 870                            "%s", _("missing feature name"));
 871             goto error;
 872         }
 873
 874         if ((val = qemuMigrationCookieFlagTypeFromString(str)) < 0) {
 875             virReportError(VIR_ERR_INTERNAL_ERROR,
 876                            _("Unknown migration cookie feature %s"),
 877                            str);
 878             VIR_FREE(str);
 879             goto error;
 880         }
 881
 882         if ((flags & (1 << val)) == 0) {
 883             virReportError(VIR_ERR_INTERNAL_ERROR,
 884                            _("Unsupported migration cookie feature %s"),
 885                            str);
 886             VIR_FREE(str);
 887         }
 888         VIR_FREE(str);
 889     }
 890     VIR_FREE(nodes);
 891
 892     if ((flags & QEMU_MIGRATION_COOKIE_GRAPHICS) &&
 893         virXPathBoolean("count(./graphics) > 0", ctxt) &&
 894         (!(mig->graphics = qemuMigrationCookieGraphicsXMLParse(ctxt))))
 895         goto error;
 896
 897     if ((flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) &&
 898         virXPathBoolean("count(./lockstate) > 0", ctxt)) {
 899         mig->lockDriver = virXPathString("string(./lockstate[1]/@driver)", ctxt);
 900         if (!mig->lockDriver) {
 901             virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
 902                            _("Missing lock driver name in migration cookie"));
 903             goto error;
 904         }
 905         mig->lockState = virXPathString("string(./lockstate[1]/leases[1])", ctxt);
 906         if (mig->lockState && STREQ(mig->lockState, ""))
 907             VIR_FREE(mig->lockState);
 908     }
 909
 910     if ((flags & QEMU_MIGRATION_COOKIE_PERSISTENT) &&
 911         virXPathBoolean("count(./domain) > 0", ctxt)) {
 912         if ((n = virXPathNodeSet("./domain", ctxt, &nodes)) > 1) {
 913             virReportError(VIR_ERR_INTERNAL_ERROR,
 914                            _("Too many domain elements in "
 915                              "migration cookie: %d"),
 916                            n);
 917             goto error;
 918         }
 919         mig->persistent = virDomainDefParseNode(doc, nodes[0],
 920                                                 caps, driver->xmlopt,
 921                                                 -1, VIR_DOMAIN_XML_INACTIVE);
 922         if (!mig->persistent) {
 923             /* virDomainDefParseNode already reported
 924              * an error for us */
 925             goto error;
 926         }
 927         VIR_FREE(nodes);
 928     }
 929
 930     if ((flags & QEMU_MIGRATION_COOKIE_NETWORK) &&
 931         virXPathBoolean("count(./network) > 0", ctxt) &&
 932         (!(mig->network = qemuMigrationCookieNetworkXMLParse(ctxt))))
 933         goto error;
 934
 935     if (flags & QEMU_MIGRATION_COOKIE_NBD &&
 936         virXPathBoolean("boolean(./nbd)", ctxt)) {
 937         char *port;
 938
 939         if (VIR_ALLOC(mig->nbd) < 0)
 940             goto error;
 941
 942         port = virXPathString("string(./nbd/@port)", ctxt);
 943         if (port && virStrToLong_i(port, NULL, 10, &mig->nbd->port) < 0) {
 944             virReportError(VIR_ERR_INTERNAL_ERROR,
 945                            _("Malformed nbd port '%s'"),
 946                            port);
 947             VIR_FREE(port);
 948             goto error;
 949         }
 950         VIR_FREE(port);
 951     }
 952
 953     virObjectUnref(caps);
 954     return 0;
 955
 956  error:
 957     VIR_FREE(tmp);
 958     VIR_FREE(nodes);
 959     virObjectUnref(caps);
 960     return -1;
 961 }
 962
 963
 964 static int
 965 qemuMigrationCookieXMLParseStr(qemuMigrationCookiePtr mig,
 966                                virQEMUDriverPtr driver,
 967                                const char *xml,
 968                                unsigned int flags)
 969 {
 970     xmlDocPtr doc = NULL;
 971     xmlXPathContextPtr ctxt = NULL;
 972     int ret = -1;
 973
 974     VIR_DEBUG("xml=%s", NULLSTR(xml));
 975
 976     if (!(doc = virXMLParseStringCtxt(xml, _("(qemu_migration_cookie)"), &ctxt)))
 977         goto cleanup;
 978
 979     ret = qemuMigrationCookieXMLParse(mig, driver, doc, ctxt, flags);
 980
 981  cleanup:
 982     xmlXPathFreeContext(ctxt);
 983     xmlFreeDoc(doc);
 984
 985     return ret;
 986 }
 987
 988
 989 static int
 990 qemuMigrationBakeCookie(qemuMigrationCookiePtr mig,
 991                         virQEMUDriverPtr driver,
 992                         virDomainObjPtr dom,
 993                         char **cookieout,
 994                         int *cookieoutlen,
 995                         unsigned int flags)
 996 {
 997     if (!cookieout || !cookieoutlen)
 998         return 0;
 999
1000     *cookieoutlen = 0;
1001
1002     if (flags & QEMU_MIGRATION_COOKIE_GRAPHICS &&
1003         qemuMigrationCookieAddGraphics(mig, driver, dom) < 0)
1004         return -1;
1005
1006     if (flags & QEMU_MIGRATION_COOKIE_LOCKSTATE &&
1007         qemuMigrationCookieAddLockstate(mig, driver, dom) < 0)
1008         return -1;
1009
1010     if (flags & QEMU_MIGRATION_COOKIE_PERSISTENT &&
1011         qemuMigrationCookieAddPersistent(mig, dom) < 0)
1012         return -1;
1013
1014     if (flags & QEMU_MIGRATION_COOKIE_NETWORK &&
1015         qemuMigrationCookieAddNetwork(mig, driver, dom) < 0) {
1016         return -1;
1017     }
1018
1019     if ((flags & QEMU_MIGRATION_COOKIE_NBD) &&
1020         qemuMigrationCookieAddNBD(mig, driver, dom) < 0)
1021         return -1;
1022
1023     if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig)))
1024         return -1;
1025
1026     *cookieoutlen = strlen(*cookieout) + 1;
1027
1028     VIR_DEBUG("cookielen=%d cookie=%s", *cookieoutlen, *cookieout);
1029
1030     return 0;
1031 }
1032
1033
1034 static qemuMigrationCookiePtr
1035 qemuMigrationEatCookie(virQEMUDriverPtr driver,
1036                        virDomainObjPtr dom,
1037                        const char *cookiein,
1038                        int cookieinlen,
1039                        unsigned int flags)
1040 {
1041     qemuMigrationCookiePtr mig = NULL;
1042
1043     /* Parse & validate incoming cookie (if any) */
1044     if (cookiein && cookieinlen &&
1045         cookiein[cookieinlen-1] != '\0') {
1046         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
1047                        _("Migration cookie was not NULL terminated"));
1048         goto error;
1049     }
1050
1051     VIR_DEBUG("cookielen=%d cookie='%s'", cookieinlen, NULLSTR(cookiein));
1052
1053     if (!(mig = qemuMigrationCookieNew(dom)))
1054         return NULL;
1055
1056     if (cookiein && cookieinlen &&
1057         qemuMigrationCookieXMLParseStr(mig,
1058                                        driver,
1059                                        cookiein,
1060                                        flags) < 0)
1061         goto error;
1062
1063     if (mig->flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) {
1064         if (!mig->lockDriver) {
1065             if (virLockManagerPluginUsesState(driver->lockManager)) {
1066                 virReportError(VIR_ERR_INTERNAL_ERROR,
1067                                _("Missing %s lock state for migration cookie"),
1068                                virLockManagerPluginGetName(driver->lockManager));
1069                 goto error;
1070             }
1071         } else if (STRNEQ(mig->lockDriver,
1072                           virLockManagerPluginGetName(driver->lockManager))) {
1073             virReportError(VIR_ERR_INTERNAL_ERROR,
1074                            _("Source host lock driver %s different from target %s"),
1075                            mig->lockDriver,
1076                            virLockManagerPluginGetName(driver->lockManager));
1077             goto error;
1078         }
1079     }
1080
1081     return mig;
1082
1083  error:
1084     qemuMigrationCookieFree(mig);
1085     return NULL;
1086 }
1087
1088 static void
1089 qemuMigrationStoreDomainState(virDomainObjPtr vm)
1090 {
1091     qemuDomainObjPrivatePtr priv = vm->privateData;
1092     priv->preMigrationState = virDomainObjGetState(vm, NULL);
1093
1094     VIR_DEBUG("Storing pre-migration state=%d domain=%p",
1095               priv->preMigrationState, vm);
1096 }
1097
1098 /* Returns true if the domain was resumed, false otherwise */
1099 static bool
1100 qemuMigrationRestoreDomainState(virConnectPtr conn, virDomainObjPtr vm)
1101 {
1102     virQEMUDriverPtr driver = conn->privateData;
1103     qemuDomainObjPrivatePtr priv = vm->privateData;
1104     int state = virDomainObjGetState(vm, NULL);
1105     bool ret = false;
1106
1107     VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%d, state=%d",
1108               driver, vm, priv->preMigrationState, state);
1109
1110     if (state == VIR_DOMAIN_PAUSED &&
1111         priv->preMigrationState == VIR_DOMAIN_RUNNING) {
1112         /* This is basically the only restore possibility that's safe
1113          * and we should attempt to do */
1114
1115         VIR_DEBUG("Restoring pre-migration state due to migration error");
1116
1117         /* we got here through some sort of failure; start the domain again */
1118         if (qemuProcessStartCPUs(driver, vm, conn,
1119                                  VIR_DOMAIN_RUNNING_MIGRATION_CANCELED,
1120                                  QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) {
1121             /* Hm, we already know we are in error here.  We don't want to
1122              * overwrite the previous error, though, so we just throw something
1123              * to the logs and hope for the best */
1124             VIR_ERROR(_("Failed to resume guest %s after failure"), vm->def->name);
1125             goto cleanup;
1126         }
1127         ret = true;
1128     }
1129
1130  cleanup:
1131     priv->preMigrationState = VIR_DOMAIN_NOSTATE;
1132     return ret;
1133 }
1134
1135 /**
1136  * qemuMigrationStartNBDServer:
1137  * @driver: qemu driver
1138  * @vm: domain
1139  *
1140  * Starts NBD server. This is a newer method to copy
1141  * storage during migration than using 'blk' and 'inc'
1142  * arguments in 'migrate' monitor command.
1143  * Error is reported here.
1144  *
1145  * Returns 0 on success, -1 otherwise.
1146  */
1147 static int
1148 qemuMigrationStartNBDServer(virQEMUDriverPtr driver,
1149                             virDomainObjPtr vm,
1150                             const char *listenAddr)
1151 {
1152     int ret = -1;
1153     qemuDomainObjPrivatePtr priv = vm->privateData;
1154     unsigned short port = 0;
1155     char *diskAlias = NULL;
1156     size_t i;
1157
1158     for (i = 0; i < vm->def->ndisks; i++) {
1159         virDomainDiskDefPtr disk = vm->def->disks[i];
1160
1161         /* skip shared, RO and source-less disks */
1162         if (disk->shared || disk->readonly || !virDomainDiskGetSource(disk))
1163             continue;
1164
1165         VIR_FREE(diskAlias);
1166         if (virAsprintf(&diskAlias, "%s%s",
1167                         QEMU_DRIVE_HOST_PREFIX, disk->info.alias) < 0)
1168             goto cleanup;
1169
1170         if (qemuDomainObjEnterMonitorAsync(driver, vm,
1171                                            QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
1172             goto cleanup;
1173
1174         if (!port &&
1175             ((virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) ||
1176              (qemuMonitorNBDServerStart(priv->mon, listenAddr, port) < 0))) {
1177             qemuDomainObjExitMonitor(driver, vm);
1178             goto cleanup;
1179         }
1180
1181         if (qemuMonitorNBDServerAdd(priv->mon, diskAlias, true) < 0) {
1182             qemuDomainObjExitMonitor(driver, vm);
1183             goto cleanup;
1184         }
1185         qemuDomainObjExitMonitor(driver, vm);
1186     }
1187
1188     priv->nbdPort = port;
1189     ret = 0;
1190
1191  cleanup:
1192     VIR_FREE(diskAlias);
1193     if (ret < 0)
1194         virPortAllocatorRelease(driver->remotePorts, port);
1195     return ret;
1196 }
1197
1198 /**
1199  * qemuMigrationDriveMirror:
1200  * @driver: qemu driver
1201  * @vm: domain
1202  * @mig: migration cookie
1203  * @host: where are we migrating to
1204  * @speed: how much should the copying be limited
1205  * @migrate_flags: migrate monitor command flags
1206  *
1207  * Run drive-mirror to feed NBD server running on dst and wait
1208  * till the process switches into another phase where writes go
1209  * simultaneously to both source and destination. And this switch
1210  * is what we are waiting for before proceeding with the next
1211  * disk. On success, update @migrate_flags so we don't tell
1212  * 'migrate' command to do the very same operation.
1213  *
1214  * Returns 0 on success (@migrate_flags updated),
1215  *        -1 otherwise.
1216  */
1217 static int
1218 qemuMigrationDriveMirror(virQEMUDriverPtr driver,
1219                          virDomainObjPtr vm,
1220                          qemuMigrationCookiePtr mig,
1221                          const char *host,
1222                          unsigned long speed,
1223                          unsigned int *migrate_flags)
1224 {
1225     qemuDomainObjPrivatePtr priv = vm->privateData;
1226     int ret = -1;
1227     int mon_ret;
1228     int port;
1229     size_t i, lastGood = 0;
1230     char *diskAlias = NULL;
1231     char *nbd_dest = NULL;
1232     char *hoststr = NULL;
1233     unsigned int mirror_flags = VIR_DOMAIN_BLOCK_REBASE_REUSE_EXT;
1234     virErrorPtr err = NULL;
1235
1236     if (!(*migrate_flags & (QEMU_MONITOR_MIGRATE_NON_SHARED_DISK |
1237                             QEMU_MONITOR_MIGRATE_NON_SHARED_INC)))
1238         return 0;
1239
1240     if (!mig->nbd) {
1241         /* Destination doesn't support NBD server.
1242          * Fall back to previous implementation. */
1243         VIR_DEBUG("Destination doesn't support NBD server "
1244                   "Falling back to previous implementation.");
1245         return 0;
1246     }
1247
1248     /* steal NBD port and thus prevent its propagation back to destination */
1249     port = mig->nbd->port;
1250     mig->nbd->port = 0;
1251
1252     /* escape literal IPv6 address */
1253     if (strchr(host, ':')) {
1254         if (virAsprintf(&hoststr, "[%s]", host) < 0)
1255             goto error;
1256     } else if (VIR_STRDUP(hoststr, host) < 0) {
1257         goto error;
1258     }
1259
1260     if (*migrate_flags & QEMU_MONITOR_MIGRATE_NON_SHARED_INC)
1261         mirror_flags |= VIR_DOMAIN_BLOCK_REBASE_SHALLOW;
1262
1263     for (i = 0; i < vm->def->ndisks; i++) {
1264         virDomainDiskDefPtr disk = vm->def->disks[i];
1265         virDomainBlockJobInfo info;
1266
1267         /* skip shared, RO and source-less disks */
1268         if (disk->shared || disk->readonly || !virDomainDiskGetSource(disk))
1269             continue;
1270
1271         VIR_FREE(diskAlias);
1272         VIR_FREE(nbd_dest);
1273         if ((virAsprintf(&diskAlias, "%s%s",
1274                          QEMU_DRIVE_HOST_PREFIX, disk->info.alias) < 0) ||
1275             (virAsprintf(&nbd_dest, "nbd:%s:%d:exportname=%s",
1276                          hoststr, port, diskAlias) < 0))
1277             goto error;
1278
1279         if (qemuDomainObjEnterMonitorAsync(driver, vm,
1280                                            QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
1281             goto error;
1282         mon_ret = qemuMonitorDriveMirror(priv->mon, diskAlias, nbd_dest,
1283                                          NULL, speed, mirror_flags);
1284         qemuDomainObjExitMonitor(driver, vm);
1285
1286         if (mon_ret < 0)
1287             goto error;
1288
1289         lastGood = i;
1290
1291         /* wait for completion */
1292         while (true) {
1293             /* Poll every 500ms for progress & to allow cancellation */
1294             struct timespec ts = { .tv_sec = 0, .tv_nsec = 500 * 1000 * 1000ull };
1295
1296             memset(&info, 0, sizeof(info));
1297
1298             if (qemuDomainObjEnterMonitorAsync(driver, vm,
1299                                                QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
1300                 goto error;
1301             if (priv->job.asyncAbort) {
1302                 /* explicitly do this *after* we entered the monitor,
1303                  * as this is a critical section so we are guaranteed
1304                  * priv->job.asyncAbort will not change */
1305                 qemuDomainObjExitMonitor(driver, vm);
1306                 virReportError(VIR_ERR_OPERATION_ABORTED, _("%s: %s"),
1307                                qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
1308                                _("canceled by client"));
1309                 goto error;
1310             }
1311             mon_ret = qemuMonitorBlockJob(priv->mon, diskAlias, NULL, 0,
1312                                           &info, BLOCK_JOB_INFO, true);
1313             qemuDomainObjExitMonitor(driver, vm);
1314
1315             if (mon_ret < 0)
1316                 goto error;
1317
1318             if (info.cur == info.end) {
1319                 VIR_DEBUG("Drive mirroring of '%s' completed", diskAlias);
1320                 break;
1321             }
1322
1323             /* XXX Frankly speaking, we should listen to the events,
1324              * instead of doing this. But this works for now and we
1325              * are doing something similar in migration itself anyway */
1326
1327             virObjectUnlock(vm);
1328
1329             nanosleep(&ts, NULL);
1330
1331             virObjectLock(vm);
1332         }
1333     }
1334
1335     /* Okay, copied. Modify migrate_flags */
1336     *migrate_flags &= ~(QEMU_MONITOR_MIGRATE_NON_SHARED_DISK |
1337                         QEMU_MONITOR_MIGRATE_NON_SHARED_INC);
1338     ret = 0;
1339
1340  cleanup:
1341     VIR_FREE(diskAlias);
1342     VIR_FREE(nbd_dest);
1343     VIR_FREE(hoststr);
1344     return ret;
1345
1346  error:
1347     /* don't overwrite any errors */
1348     err = virSaveLastError();
1349     /* cancel any outstanding jobs */
1350     while (lastGood) {
1351         virDomainDiskDefPtr disk = vm->def->disks[--lastGood];
1352
1353         /* skip shared, RO disks */
1354         if (disk->shared || disk->readonly || !virDomainDiskGetSource(disk))
1355             continue;
1356
1357         VIR_FREE(diskAlias);
1358         if (virAsprintf(&diskAlias, "%s%s",
1359                         QEMU_DRIVE_HOST_PREFIX, disk->info.alias) < 0)
1360             continue;
1361         if (qemuDomainObjEnterMonitorAsync(driver, vm,
1362                                            QEMU_ASYNC_JOB_MIGRATION_OUT) == 0) {
1363             if (qemuMonitorBlockJob(priv->mon, diskAlias, NULL, 0,
1364                                     NULL, BLOCK_JOB_ABORT, true) < 0) {
1365                 VIR_WARN("Unable to cancel block-job on '%s'", diskAlias);
1366             }
1367             qemuDomainObjExitMonitor(driver, vm);
1368         } else {
1369             VIR_WARN("Unable to enter monitor. No block job cancelled");
1370         }
1371     }
1372     if (err)
1373         virSetError(err);
1374     virFreeError(err);
1375     goto cleanup;
1376 }
1377
1378
1379 static void
1380 qemuMigrationStopNBDServer(virQEMUDriverPtr driver,
1381                            virDomainObjPtr vm,
1382                            qemuMigrationCookiePtr mig)
1383 {
1384     qemuDomainObjPrivatePtr priv = vm->privateData;
1385
1386     if (!mig->nbd)
1387         return;
1388
1389     if (qemuDomainObjEnterMonitorAsync(driver, vm,
1390                                        QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
1391         return;
1392
1393     if (qemuMonitorNBDServerStop(priv->mon) < 0)
1394         VIR_WARN("Unable to stop NBD server");
1395
1396     qemuDomainObjExitMonitor(driver, vm);
1397
1398     virPortAllocatorRelease(driver->remotePorts, priv->nbdPort);
1399     priv->nbdPort = 0;
1400 }
1401
1402 static void
1403 qemuMigrationCancelDriveMirror(qemuMigrationCookiePtr mig,
1404                                virQEMUDriverPtr driver,
1405                                virDomainObjPtr vm)
1406 {
1407     qemuDomainObjPrivatePtr priv = vm->privateData;
1408     size_t i;
1409     char *diskAlias = NULL;
1410
1411     VIR_DEBUG("mig=%p nbdPort=%d", mig->nbd, priv->nbdPort);
1412
1413     for (i = 0; i < vm->def->ndisks; i++) {
1414         virDomainDiskDefPtr disk = vm->def->disks[i];
1415
1416         /* skip shared, RO and source-less disks */
1417         if (disk->shared || disk->readonly || !virDomainDiskGetSource(disk))
1418             continue;
1419
1420         VIR_FREE(diskAlias);
1421         if (virAsprintf(&diskAlias, "%s%s",
1422                         QEMU_DRIVE_HOST_PREFIX, disk->info.alias) < 0)
1423             goto cleanup;
1424
1425         if (qemuDomainObjEnterMonitorAsync(driver, vm,
1426                                            QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
1427             goto cleanup;
1428
1429         if (qemuMonitorBlockJob(priv->mon, diskAlias, NULL, 0,
1430                                 NULL, BLOCK_JOB_ABORT, true) < 0)
1431             VIR_WARN("Unable to stop block job on %s", diskAlias);
1432         qemuDomainObjExitMonitor(driver, vm);
1433     }
1434
1435  cleanup:
1436     VIR_FREE(diskAlias);
1437     return;
1438 }
1439
1440 /* Validate whether the domain is safe to migrate.  If vm is NULL,
1441  * then this is being run in the v2 Prepare stage on the destination
1442  * (where we only have the target xml); if vm is provided, then this
1443  * is being run in either v2 Perform or v3 Begin (where we also have
1444  * access to all of the domain's metadata, such as whether it is
1445  * marked autodestroy or has snapshots).  While it would be nice to
1446  * assume that checking on source is sufficient to prevent ever
1447  * talking to the destination in the first place, we are stuck with
1448  * the fact that older servers did not do checks on the source. */
1449 bool
1450 qemuMigrationIsAllowed(virQEMUDriverPtr driver, virDomainObjPtr vm,
1451                        virDomainDefPtr def, bool remote, bool abort_on_error)
1452 {
1453     int nsnapshots;
1454     int pauseReason;
1455     bool forbid;
1456     size_t i;
1457
1458     if (vm) {
1459         if (qemuProcessAutoDestroyActive(driver, vm)) {
1460             virReportError(VIR_ERR_OPERATION_INVALID,
1461                            "%s", _("domain is marked for auto destroy"));
1462             return false;
1463         }
1464
1465         /* perform these checks only when migrating to remote hosts */
1466         if (remote) {
1467             nsnapshots = virDomainSnapshotObjListNum(vm->snapshots, NULL, 0);
1468             if (nsnapshots < 0)
1469                 return false;
1470
1471             if (nsnapshots > 0) {
1472                 virReportError(VIR_ERR_OPERATION_INVALID,
1473                                _("cannot migrate domain with %d snapshots"),
1474                                nsnapshots);
1475                 return false;
1476             }
1477
1478             /* cancel migration if disk I/O error is emitted while migrating */
1479             if (abort_on_error &&
1480                 virDomainObjGetState(vm, &pauseReason) == VIR_DOMAIN_PAUSED &&
1481                 pauseReason == VIR_DOMAIN_PAUSED_IOERROR) {
1482                 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1483                                _("cannot migrate domain with I/O error"));
1484                 return false;
1485             }
1486
1487         }
1488
1489         if (virDomainHasDiskMirror(vm)) {
1490             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1491                            _("domain has an active block job"));
1492             return false;
1493         }
1494
1495         def = vm->def;
1496     }
1497
1498     /* Migration with USB host devices is allowed, all other devices are
1499      * forbidden.
1500      */
1501     forbid = false;
1502     for (i = 0; i < def->nhostdevs; i++) {
1503         virDomainHostdevDefPtr hostdev = def->hostdevs[i];
1504         if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
1505             hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) {
1506             forbid = true;
1507             break;
1508         }
1509     }
1510     if (forbid) {
1511         virReportError(VIR_ERR_OPERATION_INVALID, "%s",
1512                        _("domain has assigned non-USB host devices"));
1513         return false;
1514     }
1515
1516     return true;
1517 }
1518
1519 static bool
1520 qemuMigrationIsSafe(virDomainDefPtr def)
1521 {
1522     size_t i;
1523
1524     for (i = 0; i < def->ndisks; i++) {
1525         virDomainDiskDefPtr disk = def->disks[i];
1526         const char *src = virDomainDiskGetSource(disk);
1527
1528         /* Our code elsewhere guarantees shared disks are either readonly (in
1529          * which case cache mode doesn't matter) or used with cache=none */
1530         if (src &&
1531             !disk->shared &&
1532             !disk->readonly &&
1533             disk->cachemode != VIR_DOMAIN_DISK_CACHE_DISABLE) {
1534             int rc;
1535
1536             if (virDomainDiskGetType(disk) == VIR_STORAGE_TYPE_FILE) {
1537                 if ((rc = virFileIsSharedFS(src)) < 0)
1538                     return false;
1539                 else if (rc == 0)
1540                     continue;
1541                 if ((rc = virStorageFileIsClusterFS(src)) < 0)
1542                     return false;
1543                 else if (rc == 1)
1544                     continue;
1545             } else if (disk->src.type == VIR_STORAGE_TYPE_NETWORK &&
1546                        disk->src.protocol == VIR_STORAGE_NET_PROTOCOL_RBD) {
1547                 continue;
1548             }
1549
1550             virReportError(VIR_ERR_MIGRATE_UNSAFE, "%s",
1551                            _("Migration may lead to data corruption if disks"
1552                              " use cache != none"));
1553             return false;
1554         }
1555     }
1556
1557     return true;
1558 }
1559
1560 /** qemuMigrationSetOffline
1561  * Pause domain for non-live migration.
1562  */
1563 int
1564 qemuMigrationSetOffline(virQEMUDriverPtr driver,
1565                         virDomainObjPtr vm)
1566 {
1567     int ret;
1568     VIR_DEBUG("driver=%p vm=%p", driver, vm);
1569     ret = qemuProcessStopCPUs(driver, vm, VIR_DOMAIN_PAUSED_MIGRATION,
1570                               QEMU_ASYNC_JOB_MIGRATION_OUT);
1571     if (ret == 0) {
1572         virObjectEventPtr event;
1573
1574         event = virDomainEventLifecycleNewFromObj(vm,
1575                                          VIR_DOMAIN_EVENT_SUSPENDED,
1576                                          VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED);
1577         if (event)
1578             qemuDomainEventQueue(driver, event);
1579     }
1580
1581     return ret;
1582 }
1583
1584
1585 static int
1586 qemuMigrationSetCompression(virQEMUDriverPtr driver,
1587                             virDomainObjPtr vm,
1588                             enum qemuDomainAsyncJob job)
1589 {
1590     qemuDomainObjPrivatePtr priv = vm->privateData;
1591     int ret;
1592
1593     if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0)
1594         return -1;
1595
1596     ret = qemuMonitorGetMigrationCapability(
1597                 priv->mon,
1598                 QEMU_MONITOR_MIGRATION_CAPS_XBZRLE);
1599
1600     if (ret < 0) {
1601         goto cleanup;
1602     } else if (ret == 0) {
1603         if (job == QEMU_ASYNC_JOB_MIGRATION_IN) {
1604             virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
1605                            _("Compressed migration is not supported by "
1606                              "target QEMU binary"));
1607         } else {
1608             virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
1609                            _("Compressed migration is not supported by "
1610                              "source QEMU binary"));
1611         }
1612         ret = -1;
1613         goto cleanup;
1614     }
1615
1616     ret = qemuMonitorSetMigrationCapability(
1617                 priv->mon,
1618                 QEMU_MONITOR_MIGRATION_CAPS_XBZRLE);
1619
1620  cleanup:
1621     qemuDomainObjExitMonitor(driver, vm);
1622     return ret;
1623 }
1624
1625 static int
1626 qemuMigrationSetAutoConverge(virQEMUDriverPtr driver,
1627                              virDomainObjPtr vm,
1628                              enum qemuDomainAsyncJob job)
1629 {
1630     qemuDomainObjPrivatePtr priv = vm->privateData;
1631     int ret;
1632
1633     if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0)
1634         return -1;
1635
1636     ret = qemuMonitorGetMigrationCapability(
1637                 priv->mon,
1638                 QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE);
1639
1640     if (ret < 0) {
1641         goto cleanup;
1642     } else if (ret == 0) {
1643         virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
1644                        _("Auto-Converge is not supported by "
1645                          "QEMU binary"));
1646         ret = -1;
1647         goto cleanup;
1648     }
1649
1650     ret = qemuMonitorSetMigrationCapability(
1651                 priv->mon,
1652                 QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE);
1653
1654  cleanup:
1655     qemuDomainObjExitMonitor(driver, vm);
1656     return ret;
1657 }
1658
1659
1660 static int
1661 qemuMigrationWaitForSpice(virQEMUDriverPtr driver,
1662                           virDomainObjPtr vm)
1663 {
1664     qemuDomainObjPrivatePtr priv = vm->privateData;
1665     bool wait_for_spice = false;
1666     bool spice_migrated = false;
1667     size_t i = 0;
1668
1669     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_SEAMLESS_MIGRATION)) {
1670         for (i = 0; i < vm->def->ngraphics; i++) {
1671             if (vm->def->graphics[i]->type == VIR_DOMAIN_GRAPHICS_TYPE_SPICE) {
1672                 wait_for_spice = true;
1673                 break;
1674             }
1675         }
1676     }
1677
1678     if (!wait_for_spice)
1679         return 0;
1680
1681     while (!spice_migrated) {
1682         /* Poll every 50ms for progress & to allow cancellation */
1683         struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000 * 1000ull };
1684
1685         if (qemuDomainObjEnterMonitorAsync(driver, vm,
1686                                            QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
1687             return -1;
1688
1689         if (qemuMonitorGetSpiceMigrationStatus(priv->mon,
1690                                                &spice_migrated) < 0) {
1691             qemuDomainObjExitMonitor(driver, vm);
1692             return -1;
1693         }
1694         qemuDomainObjExitMonitor(driver, vm);
1695         virObjectUnlock(vm);
1696         nanosleep(&ts, NULL);
1697         virObjectLock(vm);
1698     }
1699
1700     return 0;
1701 }
1702
1703 static int
1704 qemuMigrationUpdateJobStatus(virQEMUDriverPtr driver,
1705                              virDomainObjPtr vm,
1706                              const char *job,
1707                              enum qemuDomainAsyncJob asyncJob)
1708 {
1709     qemuDomainObjPrivatePtr priv = vm->privateData;
1710     int ret;
1711     qemuMonitorMigrationStatus status;
1712
1713     memset(&status, 0, sizeof(status));
1714
1715     ret = qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob);
1716     if (ret < 0) {
1717         /* Guest already exited or waiting for the job timed out; nothing
1718          * further to update. */
1719         return ret;
1720     }
1721     ret = qemuMonitorGetMigrationStatus(priv->mon, &status);
1722
1723     qemuDomainObjExitMonitor(driver, vm);
1724
1725     priv->job.status = status;
1726
1727     if (ret < 0 || virTimeMillisNow(&priv->job.info.timeElapsed) < 0)
1728         return -1;
1729
1730     priv->job.info.timeElapsed -= priv->job.start;
1731
1732     ret = -1;
1733     switch (priv->job.status.status) {
1734     case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE:
1735         priv->job.info.type = VIR_DOMAIN_JOB_NONE;
1736         virReportError(VIR_ERR_OPERATION_FAILED,
1737                        _("%s: %s"), job, _("is not active"));
1738         break;
1739
1740     case QEMU_MONITOR_MIGRATION_STATUS_SETUP:
1741         ret = 0;
1742         break;
1743
1744     case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE:
1745         priv->job.info.fileTotal = priv->job.status.disk_total;
1746         priv->job.info.fileRemaining = priv->job.status.disk_remaining;
1747         priv->job.info.fileProcessed = priv->job.status.disk_transferred;
1748
1749         priv->job.info.memTotal = priv->job.status.ram_total;
1750         priv->job.info.memRemaining = priv->job.status.ram_remaining;
1751         priv->job.info.memProcessed = priv->job.status.ram_transferred;
1752
1753         priv->job.info.dataTotal =
1754             priv->job.status.ram_total + priv->job.status.disk_total;
1755         priv->job.info.dataRemaining =
1756             priv->job.status.ram_remaining + priv->job.status.disk_remaining;
1757         priv->job.info.dataProcessed =
1758             priv->job.status.ram_transferred +
1759             priv->job.status.disk_transferred;
1760
1761         ret = 0;
1762         break;
1763
1764     case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED:
1765         priv->job.info.type = VIR_DOMAIN_JOB_COMPLETED;
1766         ret = 0;
1767         break;
1768
1769     case QEMU_MONITOR_MIGRATION_STATUS_ERROR:
1770         priv->job.info.type = VIR_DOMAIN_JOB_FAILED;
1771         virReportError(VIR_ERR_OPERATION_FAILED,
1772                        _("%s: %s"), job, _("unexpectedly failed"));
1773         break;
1774
1775     case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED:
1776         priv->job.info.type = VIR_DOMAIN_JOB_CANCELLED;
1777         virReportError(VIR_ERR_OPERATION_ABORTED,
1778                        _("%s: %s"), job, _("canceled by client"));
1779         break;
1780     }
1781
1782     return ret;
1783 }
1784
1785
1786 /* Returns 0 on success, -2 when migration needs to be cancelled, or -1 when
1787  * QEMU reports failed migration.
1788  */
1789 static int
1790 qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, virDomainObjPtr vm,
1791                                enum qemuDomainAsyncJob asyncJob,
1792                                virConnectPtr dconn, bool abort_on_error)
1793 {
1794     qemuDomainObjPrivatePtr priv = vm->privateData;
1795     const char *job;
1796     int pauseReason;
1797
1798     switch (priv->job.asyncJob) {
1799     case QEMU_ASYNC_JOB_MIGRATION_OUT:
1800         job = _("migration job");
1801         break;
1802     case QEMU_ASYNC_JOB_SAVE:
1803         job = _("domain save job");
1804         break;
1805     case QEMU_ASYNC_JOB_DUMP:
1806         job = _("domain core dump job");
1807         break;
1808     default:
1809         job = _("job");
1810     }
1811
1812     priv->job.info.type = VIR_DOMAIN_JOB_UNBOUNDED;
1813
1814     while (priv->job.info.type == VIR_DOMAIN_JOB_UNBOUNDED) {
1815         /* Poll every 50ms for progress & to allow cancellation */
1816         struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000 * 1000ull };
1817
1818         if (qemuMigrationUpdateJobStatus(driver, vm, job, asyncJob) == -1)
1819             break;
1820
1821         /* cancel migration if disk I/O error is emitted while migrating */
1822         if (abort_on_error &&
1823             virDomainObjGetState(vm, &pauseReason) == VIR_DOMAIN_PAUSED &&
1824             pauseReason == VIR_DOMAIN_PAUSED_IOERROR) {
1825             virReportError(VIR_ERR_OPERATION_FAILED,
1826                            _("%s: %s"), job, _("failed due to I/O error"));
1827             break;
1828         }
1829
1830         if (dconn && virConnectIsAlive(dconn) <= 0) {
1831             virReportError(VIR_ERR_OPERATION_FAILED, "%s",
1832                            _("Lost connection to destination host"));
1833             break;
1834         }
1835
1836         virObjectUnlock(vm);
1837
1838         nanosleep(&ts, NULL);
1839
1840         virObjectLock(vm);
1841     }
1842
1843     if (priv->job.info.type == VIR_DOMAIN_JOB_COMPLETED) {
1844         return 0;
1845     } else if (priv->job.info.type == VIR_DOMAIN_JOB_UNBOUNDED) {
1846         /* The migration was aborted by us rather than QEMU itself so let's
1847          * update the job type and notify the caller to send migrate_cancel.
1848          */
1849         priv->job.info.type = VIR_DOMAIN_JOB_FAILED;
1850         return -2;
1851     } else {
1852         return -1;
1853     }
1854 }
1855
1856
1857 static int
1858 qemuDomainMigrateGraphicsRelocate(virQEMUDriverPtr driver,
1859                                   virDomainObjPtr vm,
1860                                   qemuMigrationCookiePtr cookie,
1861                                   const char *graphicsuri)
1862 {
1863     qemuDomainObjPrivatePtr priv = vm->privateData;
1864     int ret = -1;
1865     const char *listenAddress = NULL;
1866     virSocketAddr addr;
1867     virURIPtr uri = NULL;
1868     int type = -1;
1869     int port = -1;
1870     int tlsPort = -1;
1871     const char *tlsSubject = NULL;
1872
1873     if (!cookie || (!cookie->graphics && !graphicsuri))
1874         return 0;
1875
1876     if (graphicsuri && !(uri = virURIParse(graphicsuri)))
1877         goto cleanup;
1878
1879     if (cookie->graphics) {
1880         type = cookie->graphics->type;
1881
1882         listenAddress = cookie->graphics->listen;
1883
1884         if (!listenAddress ||
1885             (virSocketAddrParse(&addr, listenAddress, AF_UNSPEC) > 0 &&
1886              virSocketAddrIsWildcard(&addr)))
1887             listenAddress = cookie->remoteHostname;
1888
1889         port = cookie->graphics->port;
1890         tlsPort = cookie->graphics->tlsPort;
1891         tlsSubject = cookie->graphics->tlsSubject;
1892     }
1893
1894     if (uri) {
1895         size_t i;
1896
1897         if ((type = virDomainGraphicsTypeFromString(uri->scheme)) < 0) {
1898             virReportError(VIR_ERR_INVALID_ARG,
1899                            _("unknown graphics type %s"), uri->scheme);
1900             goto cleanup;
1901         }
1902
1903         if (uri->server)
1904             listenAddress = uri->server;
1905         if (uri->port > 0)
1906             port = uri->port;
1907
1908         for (i = 0; i < uri->paramsCount; i++) {
1909             virURIParamPtr param = uri->params + i;
1910
1911             if (STRCASEEQ(param->name, "tlsPort")) {
1912                 if (virStrToLong_i(param->value, NULL, 10, &tlsPort) < 0) {
1913                     virReportError(VIR_ERR_INVALID_ARG,
1914                                    _("invalid tlsPort number: %s"),
1915                                    param->value);
1916                     goto cleanup;
1917                 }
1918             } else if (STRCASEEQ(param->name, "tlsSubject")) {
1919                 tlsSubject = param->value;
1920             }
1921         }
1922     }
1923
1924     /* QEMU doesn't support VNC relocation yet, so
1925      * skip it to avoid generating an error
1926      */
1927     if (type != VIR_DOMAIN_GRAPHICS_TYPE_SPICE) {
1928         ret = 0;
1929         goto cleanup;
1930     }
1931
1932     if (qemuDomainObjEnterMonitorAsync(driver, vm,
1933                                        QEMU_ASYNC_JOB_MIGRATION_OUT) == 0) {
1934         ret = qemuMonitorGraphicsRelocate(priv->mon, type, listenAddress,
1935                                           port, tlsPort, tlsSubject);
1936         qemuDomainObjExitMonitor(driver, vm);
1937     }
1938
1939  cleanup:
1940     virURIFree(uri);
1941     return ret;
1942 }
1943
1944
1945 static int
1946 qemuDomainMigrateOPDRelocate(virQEMUDriverPtr driver ATTRIBUTE_UNUSED,
1947                              virDomainObjPtr vm,
1948                              qemuMigrationCookiePtr cookie)
1949 {
1950     virDomainNetDefPtr netptr;
1951     int ret = -1;
1952     size_t i;
1953
1954     for (i = 0; i < cookie->network->nnets; i++) {
1955         netptr = vm->def->nets[i];
1956
1957         switch (cookie->network->net[i].vporttype) {
1958         case VIR_NETDEV_VPORT_PROFILE_NONE:
1959         case VIR_NETDEV_VPORT_PROFILE_8021QBG:
1960         case VIR_NETDEV_VPORT_PROFILE_8021QBH:
1961            break;
1962         case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH:
1963             if (virNetDevOpenvswitchSetMigrateData(cookie->network->net[i].portdata,
1964                                                    netptr->ifname) != 0) {
1965                 virReportSystemError(VIR_ERR_INTERNAL_ERROR,
1966                                      _("Unable to run command to set OVS port data for "
1967                                      "interface %s"), netptr->ifname);
1968                 goto cleanup;
1969             }
1970             break;
1971         default:
1972             break;
1973         }
1974     }
1975
1976     ret = 0;
1977  cleanup:
1978     return ret;
1979 }
1980
1981
1982 /* This is called for outgoing non-p2p migrations when a connection to the
1983  * client which initiated the migration was closed but we were waiting for it
1984  * to follow up with the next phase, that is, in between
1985  * qemuDomainMigrateBegin3 and qemuDomainMigratePerform3 or
1986  * qemuDomainMigratePerform3 and qemuDomainMigrateConfirm3.
1987  */
1988 static virDomainObjPtr
1989 qemuMigrationCleanup(virDomainObjPtr vm,
1990                      virConnectPtr conn,
1991                      void *opaque)
1992 {
1993     virQEMUDriverPtr driver = opaque;
1994     qemuDomainObjPrivatePtr priv = vm->privateData;
1995
1996     VIR_DEBUG("vm=%s, conn=%p, asyncJob=%s, phase=%s",
1997               vm->def->name, conn,
1998               qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
1999               qemuDomainAsyncJobPhaseToString(priv->job.asyncJob,
2000                                               priv->job.phase));
2001
2002     if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
2003         goto cleanup;
2004
2005     VIR_DEBUG("The connection which started outgoing migration of domain %s"
2006               " was closed; canceling the migration",
2007               vm->def->name);
2008
2009     switch ((enum qemuMigrationJobPhase) priv->job.phase) {
2010     case QEMU_MIGRATION_PHASE_BEGIN3:
2011         /* just forget we were about to migrate */
2012         qemuDomainObjDiscardAsyncJob(driver, vm);
2013         break;
2014
2015     case QEMU_MIGRATION_PHASE_PERFORM3_DONE:
2016         VIR_WARN("Migration of domain %s finished but we don't know if the"
2017                  " domain was successfully started on destination or not",
2018                  vm->def->name);
2019         /* clear the job and let higher levels decide what to do */
2020         qemuDomainObjDiscardAsyncJob(driver, vm);
2021         break;
2022
2023     case QEMU_MIGRATION_PHASE_PERFORM3:
2024         /* cannot be seen without an active migration API; unreachable */
2025     case QEMU_MIGRATION_PHASE_CONFIRM3:
2026     case QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED:
2027         /* all done; unreachable */
2028     case QEMU_MIGRATION_PHASE_PREPARE:
2029     case QEMU_MIGRATION_PHASE_FINISH2:
2030     case QEMU_MIGRATION_PHASE_FINISH3:
2031         /* incoming migration; unreachable */
2032     case QEMU_MIGRATION_PHASE_PERFORM2:
2033         /* single phase outgoing migration; unreachable */
2034     case QEMU_MIGRATION_PHASE_NONE:
2035     case QEMU_MIGRATION_PHASE_LAST:
2036         /* unreachable */
2037         ;
2038     }
2039
2040  cleanup:
2041     return vm;
2042 }
2043
2044
2045 /* The caller is supposed to lock the vm and start a migration job. */
2046 static char
2047 *qemuMigrationBeginPhase(virQEMUDriverPtr driver,
2048                          virDomainObjPtr vm,
2049                          const char *xmlin,
2050                          const char *dname,
2051                          char **cookieout,
2052                          int *cookieoutlen,
2053                          unsigned long flags)
2054 {
2055     char *rv = NULL;
2056     qemuMigrationCookiePtr mig = NULL;
2057     virDomainDefPtr def = NULL;
2058     qemuDomainObjPrivatePtr priv = vm->privateData;
2059     virCapsPtr caps = NULL;
2060     unsigned int cookieFlags = QEMU_MIGRATION_COOKIE_LOCKSTATE;
2061     bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR);
2062
2063     VIR_DEBUG("driver=%p, vm=%p, xmlin=%s, dname=%s,"
2064               " cookieout=%p, cookieoutlen=%p, flags=%lx",
2065               driver, vm, NULLSTR(xmlin), NULLSTR(dname),
2066               cookieout, cookieoutlen, flags);
2067
2068     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
2069         goto cleanup;
2070
2071     /* Only set the phase if we are inside QEMU_ASYNC_JOB_MIGRATION_OUT.
2072      * Otherwise we will start the async job later in the perform phase losing
2073      * change protection.
2074      */
2075     if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT)
2076         qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_BEGIN3);
2077
2078     if (!qemuMigrationIsAllowed(driver, vm, NULL, true, abort_on_error))
2079         goto cleanup;
2080
2081     if (!(flags & VIR_MIGRATE_UNSAFE) && !qemuMigrationIsSafe(vm->def))
2082         goto cleanup;
2083
2084     if (flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC) &&
2085         virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DRIVE_MIRROR)) {
2086         /* TODO support NBD for TUNNELLED migration */
2087         if (flags & VIR_MIGRATE_TUNNELLED) {
2088             VIR_WARN("NBD in tunnelled migration is currently not supported");
2089         } else {
2090             cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
2091             priv->nbdPort = 0;
2092         }
2093     }
2094
2095     if (!(mig = qemuMigrationEatCookie(driver, vm, NULL, 0, 0)))
2096         goto cleanup;
2097
2098     if (qemuMigrationBakeCookie(mig, driver, vm,
2099                                 cookieout, cookieoutlen,
2100                                 cookieFlags) < 0)
2101         goto cleanup;
2102
2103     if (flags & VIR_MIGRATE_OFFLINE) {
2104         if (flags & (VIR_MIGRATE_NON_SHARED_DISK |
2105                      VIR_MIGRATE_NON_SHARED_INC)) {
2106             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2107                            _("offline migration cannot handle "
2108                              "non-shared storage"));
2109             goto cleanup;
2110         }
2111         if (!(flags & VIR_MIGRATE_PERSIST_DEST)) {
2112             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2113                            _("offline migration must be specified with "
2114                              "the persistent flag set"));
2115             goto cleanup;
2116         }
2117         if (flags & VIR_MIGRATE_TUNNELLED) {
2118             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2119                            _("tunnelled offline migration does not "
2120                              "make sense"));
2121             goto cleanup;
2122         }
2123     }
2124
2125     if (xmlin) {
2126         if (!(def = virDomainDefParseString(xmlin, caps, driver->xmlopt,
2127                                             QEMU_EXPECTED_VIRT_TYPES,
2128                                             VIR_DOMAIN_XML_INACTIVE)))
2129             goto cleanup;
2130
2131         if (!qemuDomainDefCheckABIStability(driver, vm->def, def))
2132             goto cleanup;
2133
2134         rv = qemuDomainDefFormatLive(driver, def, false, true);
2135     } else {
2136         rv = qemuDomainDefFormatLive(driver, vm->def, false, true);
2137     }
2138
2139  cleanup:
2140     qemuMigrationCookieFree(mig);
2141     virObjectUnref(caps);
2142     virDomainDefFree(def);
2143     return rv;
2144 }
2145
2146 char *
2147 qemuMigrationBegin(virConnectPtr conn,
2148                    virDomainObjPtr vm,
2149                    const char *xmlin,
2150                    const char *dname,
2151                    char **cookieout,
2152                    int *cookieoutlen,
2153                    unsigned long flags)
2154 {
2155     virQEMUDriverPtr driver = conn->privateData;
2156     char *xml = NULL;
2157     enum qemuDomainAsyncJob asyncJob;
2158
2159     if ((flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
2160         if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
2161             goto cleanup;
2162         asyncJob = QEMU_ASYNC_JOB_MIGRATION_OUT;
2163     } else {
2164         if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0)
2165             goto cleanup;
2166         asyncJob = QEMU_ASYNC_JOB_NONE;
2167     }
2168
2169     qemuMigrationStoreDomainState(vm);
2170
2171     if (!virDomainObjIsActive(vm) && !(flags & VIR_MIGRATE_OFFLINE)) {
2172         virReportError(VIR_ERR_OPERATION_INVALID,
2173                        "%s", _("domain is not running"));
2174         goto endjob;
2175     }
2176
2177     /* Check if there is any ejected media.
2178      * We don't want to require them on the destination.
2179      */
2180     if (!(flags & VIR_MIGRATE_OFFLINE) &&
2181         qemuDomainCheckEjectableMedia(driver, vm, asyncJob) < 0)
2182         goto endjob;
2183
2184     if (!(xml = qemuMigrationBeginPhase(driver, vm, xmlin, dname,
2185                                         cookieout, cookieoutlen,
2186                                         flags)))
2187         goto endjob;
2188
2189     if ((flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
2190         /* We keep the job active across API calls until the confirm() call.
2191          * This prevents any other APIs being invoked while migration is taking
2192          * place.
2193          */
2194         if (virCloseCallbacksSet(driver->closeCallbacks, vm, conn,
2195                                  qemuMigrationCleanup) < 0)
2196             goto endjob;
2197         if (qemuMigrationJobContinue(vm) == 0) {
2198             vm = NULL;
2199             virReportError(VIR_ERR_OPERATION_FAILED,
2200                            "%s", _("domain disappeared"));
2201             VIR_FREE(xml);
2202             if (cookieout)
2203                 VIR_FREE(*cookieout);
2204         }
2205     } else {
2206         goto endjob;
2207     }
2208
2209  cleanup:
2210     if (vm)
2211         virObjectUnlock(vm);
2212     return xml;
2213
2214  endjob:
2215     if ((flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
2216         if (qemuMigrationJobFinish(driver, vm) == 0)
2217             vm = NULL;
2218     } else {
2219         if (!qemuDomainObjEndJob(driver, vm))
2220             vm = NULL;
2221     }
2222     goto cleanup;
2223 }
2224
2225
2226 /* Prepare is the first step, and it runs on the destination host.
2227  */
2228
2229 static void
2230 qemuMigrationPrepareCleanup(virQEMUDriverPtr driver,
2231                             virDomainObjPtr vm)
2232 {
2233     qemuDomainObjPrivatePtr priv = vm->privateData;
2234
2235     VIR_DEBUG("driver=%p, vm=%s, job=%s, asyncJob=%s",
2236               driver,
2237               vm->def->name,
2238               qemuDomainJobTypeToString(priv->job.active),
2239               qemuDomainAsyncJobTypeToString(priv->job.asyncJob));
2240
2241     virPortAllocatorRelease(driver->migrationPorts, priv->migrationPort);
2242     priv->migrationPort = 0;
2243
2244     if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_IN))
2245         return;
2246     qemuDomainObjDiscardAsyncJob(driver, vm);
2247 }
2248
2249 static int
2250 qemuMigrationPrepareAny(virQEMUDriverPtr driver,
2251                         virConnectPtr dconn,
2252                         const char *cookiein,
2253                         int cookieinlen,
2254                         char **cookieout,
2255                         int *cookieoutlen,
2256                         virDomainDefPtr *def,
2257                         const char *origname,
2258                         virStreamPtr st,
2259                         unsigned short port,
2260                         bool autoPort,
2261                         const char *listenAddress,
2262                         unsigned long flags)
2263 {
2264     virDomainObjPtr vm = NULL;
2265     virObjectEventPtr event = NULL;
2266     int ret = -1;
2267     int dataFD[2] = { -1, -1 };
2268     qemuDomainObjPrivatePtr priv = NULL;
2269     unsigned long long now;
2270     qemuMigrationCookiePtr mig = NULL;
2271     bool tunnel = !!st;
2272     char *xmlout = NULL;
2273     unsigned int cookieFlags;
2274     virCapsPtr caps = NULL;
2275     char *migrateFrom = NULL;
2276     bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR);
2277     bool taint_hook = false;
2278
2279     if (virTimeMillisNow(&now) < 0)
2280         return -1;
2281
2282     if (flags & VIR_MIGRATE_OFFLINE) {
2283         if (flags & (VIR_MIGRATE_NON_SHARED_DISK |
2284                      VIR_MIGRATE_NON_SHARED_INC)) {
2285             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2286                            _("offline migration cannot handle "
2287                              "non-shared storage"));
2288             goto cleanup;
2289         }
2290         if (!(flags & VIR_MIGRATE_PERSIST_DEST)) {
2291             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2292                            _("offline migration must be specified with "
2293                              "the persistent flag set"));
2294             goto cleanup;
2295         }
2296         if (tunnel) {
2297             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
2298                            _("tunnelled offline migration does not "
2299                              "make sense"));
2300             goto cleanup;
2301         }
2302     }
2303
2304     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
2305         goto cleanup;
2306
2307     if (!qemuMigrationIsAllowed(driver, NULL, *def, true, abort_on_error))
2308         goto cleanup;
2309
2310     /* Let migration hook filter domain XML */
2311     if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) {
2312         char *xml;
2313         int hookret;
2314
2315         if (!(xml = qemuDomainDefFormatXML(driver, *def,
2316                                            VIR_DOMAIN_XML_SECURE |
2317                                            VIR_DOMAIN_XML_MIGRATABLE)))
2318             goto cleanup;
2319
2320         hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, (*def)->name,
2321                               VIR_HOOK_QEMU_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN,
2322                               NULL, xml, &xmlout);
2323         VIR_FREE(xml);
2324
2325         if (hookret < 0) {
2326             goto cleanup;
2327         } else if (hookret == 0) {
2328             if (!*xmlout) {
2329                 VIR_DEBUG("Migrate hook filter returned nothing; using the"
2330                           " original XML");
2331             } else {
2332                 virDomainDefPtr newdef;
2333
2334                 VIR_DEBUG("Using hook-filtered domain XML: %s", xmlout);
2335                 newdef = virDomainDefParseString(xmlout, caps, driver->xmlopt,
2336                                                  QEMU_EXPECTED_VIRT_TYPES,
2337                                                  VIR_DOMAIN_XML_INACTIVE);
2338                 if (!newdef)
2339                     goto cleanup;
2340
2341                 if (!qemuDomainDefCheckABIStability(driver, *def, newdef)) {
2342                     virDomainDefFree(newdef);
2343                     goto cleanup;
2344                 }
2345
2346                 virDomainDefFree(*def);
2347                 *def = newdef;
2348                 /* We should taint the domain here. However, @vm and therefore
2349                  * privateData too are still NULL, so just notice the fact and
2350                  * taint it later. */
2351                 taint_hook = true;
2352             }
2353         }
2354     }
2355
2356     if (tunnel) {
2357         /* QEMU will be started with -incoming stdio
2358          * (which qemu_command might convert to exec:cat or fd:n)
2359          */
2360         if (VIR_STRDUP(migrateFrom, "stdio") < 0)
2361             goto cleanup;
2362     } else {
2363         virSocketAddr listenAddressSocket;
2364         bool encloseAddress = false;
2365         bool hostIPv6Capable = false;
2366         bool qemuIPv6Capable = false;
2367         virQEMUCapsPtr qemuCaps = NULL;
2368         struct addrinfo *info = NULL;
2369         struct addrinfo hints = { .ai_flags = AI_ADDRCONFIG,
2370                                   .ai_socktype = SOCK_STREAM };
2371
2372         if (getaddrinfo("::", NULL, &hints, &info) == 0) {
2373             freeaddrinfo(info);
2374             hostIPv6Capable = true;
2375         }
2376         if (!(qemuCaps = virQEMUCapsCacheLookupCopy(driver->qemuCapsCache,
2377                                                     (*def)->emulator)))
2378             goto cleanup;
2379
2380         qemuIPv6Capable = virQEMUCapsGet(qemuCaps, QEMU_CAPS_IPV6_MIGRATION);
2381         virObjectUnref(qemuCaps);
2382
2383         if (listenAddress) {
2384             if (virSocketAddrIsNumeric(listenAddress)) {
2385                 /* listenAddress is numeric IPv4 or IPv6 */
2386                 if (virSocketAddrParse(&listenAddressSocket, listenAddress, AF_UNSPEC) < 0)
2387                     goto cleanup;
2388
2389                 /* address parsed successfully */
2390                 if (VIR_SOCKET_ADDR_IS_FAMILY(&listenAddressSocket, AF_INET6)) {
2391                     if (!qemuIPv6Capable) {
2392                         virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2393                                        _("qemu isn't capable of IPv6"));
2394                         goto cleanup;
2395                     }
2396                     if (!hostIPv6Capable) {
2397                         virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
2398                                        _("host isn't capable of IPv6"));
2399                         goto cleanup;
2400                     }
2401                     /* IPv6 address must be escaped in brackets on the cmd line */
2402                     encloseAddress = true;
2403                 }
2404             } else {
2405                 /* listenAddress is a hostname */
2406             }
2407         } else {
2408             /* Listen on :: instead of 0.0.0.0 if QEMU understands it
2409              * and there is at least one IPv6 address configured
2410              */
2411             listenAddress = qemuIPv6Capable && hostIPv6Capable ?
2412                 encloseAddress = true, "::" : "0.0.0.0";
2413         }
2414
2415         /* QEMU will be started with -incoming [<IPv6 addr>]:port,
2416          * -incoming <IPv4 addr>:port or -incoming <hostname>:port
2417          */
2418         if ((encloseAddress &&
2419              virAsprintf(&migrateFrom, "tcp:[%s]:%d", listenAddress, port) < 0) ||
2420             (!encloseAddress &&
2421              virAsprintf(&migrateFrom, "tcp:%s:%d", listenAddress, port) < 0))
2422             goto cleanup;
2423     }
2424
2425     if (!(vm = virDomainObjListAdd(driver->domains, *def,
2426                                    driver->xmlopt,
2427                                    VIR_DOMAIN_OBJ_LIST_ADD_LIVE |
2428                                    VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE,
2429                                    NULL)))
2430         goto cleanup;
2431
2432     *def = NULL;
2433     priv = vm->privateData;
2434     if (VIR_STRDUP(priv->origname, origname) < 0)
2435         goto cleanup;
2436
2437     if (taint_hook) {
2438         /* Domain XML has been altered by a hook script. */
2439         priv->hookRun = true;
2440     }
2441
2442     if (!(mig = qemuMigrationEatCookie(driver, vm, cookiein, cookieinlen,
2443                                        QEMU_MIGRATION_COOKIE_LOCKSTATE |
2444                                        QEMU_MIGRATION_COOKIE_NBD)))
2445         goto cleanup;
2446
2447     if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
2448         goto cleanup;
2449     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);
2450
2451     /* Domain starts inactive, even if the domain XML had an id field. */
2452     vm->def->id = -1;
2453
2454     if (flags & VIR_MIGRATE_OFFLINE)
2455         goto done;
2456
2457     if (tunnel &&
2458         (pipe(dataFD) < 0 || virSetCloseExec(dataFD[1]) < 0)) {
2459         virReportSystemError(errno, "%s",
2460                              _("cannot create pipe for tunnelled migration"));
2461         goto endjob;
2462     }
2463
2464     /* Start the QEMU daemon, with the same command-line arguments plus
2465      * -incoming $migrateFrom
2466      */
2467     if (qemuProcessStart(dconn, driver, vm, migrateFrom, dataFD[0], NULL, NULL,
2468                          VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_START,
2469                          VIR_QEMU_PROCESS_START_PAUSED |
2470                          VIR_QEMU_PROCESS_START_AUTODESTROY) < 0) {
2471         virDomainAuditStart(vm, "migrated", false);
2472         goto endjob;
2473     }
2474
2475     if (tunnel) {
2476         if (virFDStreamOpen(st, dataFD[1]) < 0) {
2477             virReportSystemError(errno, "%s",
2478                                  _("cannot pass pipe for tunnelled migration"));
2479             goto stop;
2480         }
2481         dataFD[1] = -1; /* 'st' owns the FD now & will close it */
2482     }
2483
2484     if (flags & VIR_MIGRATE_COMPRESSED &&
2485         qemuMigrationSetCompression(driver, vm,
2486                                     QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
2487         goto stop;
2488
2489     if (mig->lockState) {
2490         VIR_DEBUG("Received lockstate %s", mig->lockState);
2491         VIR_FREE(priv->lockState);
2492         priv->lockState = mig->lockState;
2493         mig->lockState = NULL;
2494     } else {
2495         VIR_DEBUG("Received no lockstate");
2496     }
2497
2498  done:
2499     if (flags & VIR_MIGRATE_OFFLINE)
2500         cookieFlags = 0;
2501     else
2502         cookieFlags = QEMU_MIGRATION_COOKIE_GRAPHICS;
2503
2504     if (mig->nbd &&
2505         flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC) &&
2506         virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_NBD_SERVER)) {
2507         if (qemuMigrationStartNBDServer(driver, vm, listenAddress) < 0) {
2508             /* error already reported */
2509             goto endjob;
2510         }
2511         cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
2512     }
2513
2514     if (qemuMigrationBakeCookie(mig, driver, vm, cookieout,
2515                                 cookieoutlen, cookieFlags) < 0) {
2516         /* We could tear down the whole guest here, but
2517          * cookie data is (so far) non-critical, so that
2518          * seems a little harsh. We'll just warn for now.
2519          */
2520         VIR_WARN("Unable to encode migration cookie");
2521     }
2522
2523     if (qemuDomainCleanupAdd(vm, qemuMigrationPrepareCleanup) < 0)
2524         goto endjob;
2525
2526     if (!(flags & VIR_MIGRATE_OFFLINE)) {
2527         virDomainAuditStart(vm, "migrated", true);
2528         event = virDomainEventLifecycleNewFromObj(vm,
2529                                          VIR_DOMAIN_EVENT_STARTED,
2530                                          VIR_DOMAIN_EVENT_STARTED_MIGRATED);
2531     }
2532
2533     /* We keep the job active across API calls until the finish() call.
2534      * This prevents any other APIs being invoked while incoming
2535      * migration is taking place.
2536      */
2537     if (!qemuMigrationJobContinue(vm)) {
2538         vm = NULL;
2539         virReportError(VIR_ERR_OPERATION_FAILED,
2540                        "%s", _("domain disappeared"));
2541         goto cleanup;
2542     }
2543
2544     if (autoPort)
2545         priv->migrationPort = port;
2546     ret = 0;
2547
2548  cleanup:
2549     VIR_FREE(migrateFrom);
2550     VIR_FREE(xmlout);
2551     VIR_FORCE_CLOSE(dataFD[0]);
2552     VIR_FORCE_CLOSE(dataFD[1]);
2553     if (vm) {
2554         if (ret < 0) {
2555             virPortAllocatorRelease(driver->remotePorts, priv->nbdPort);
2556             priv->nbdPort = 0;
2557         }
2558         if (ret >= 0 || vm->persistent)
2559             virObjectUnlock(vm);
2560         else
2561             qemuDomainRemoveInactive(driver, vm);
2562     }
2563     if (event)
2564         qemuDomainEventQueue(driver, event);
2565     qemuMigrationCookieFree(mig);
2566     virObjectUnref(caps);
2567     return ret;
2568
2569  stop:
2570     virDomainAuditStart(vm, "migrated", false);
2571     qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, 0);
2572
2573  endjob:
2574     if (!qemuMigrationJobFinish(driver, vm)) {
2575         vm = NULL;
2576     }
2577     goto cleanup;
2578 }
2579
2580
2581 /*
2582  * This version starts an empty VM listening on a localhost TCP port, and
2583  * sets up the corresponding virStream to handle the incoming data.
2584  */
2585 int
2586 qemuMigrationPrepareTunnel(virQEMUDriverPtr driver,
2587                            virConnectPtr dconn,
2588                            const char *cookiein,
2589                            int cookieinlen,
2590                            char **cookieout,
2591                            int *cookieoutlen,
2592                            virStreamPtr st,
2593                            virDomainDefPtr *def,
2594                            const char *origname,
2595                            unsigned long flags)
2596 {
2597     int ret;
2598
2599     VIR_DEBUG("driver=%p, dconn=%p, cookiein=%s, cookieinlen=%d, "
2600               "cookieout=%p, cookieoutlen=%p, st=%p, def=%p, "
2601               "origname=%s, flags=%lx",
2602               driver, dconn, NULLSTR(cookiein), cookieinlen,
2603               cookieout, cookieoutlen, st, *def, origname, flags);
2604
2605     if (st == NULL) {
2606         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
2607                        _("tunnelled migration requested but NULL stream passed"));
2608         return -1;
2609     }
2610
2611     ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen,
2612                                   cookieout, cookieoutlen, def, origname,
2613                                   st, 0, false, NULL, flags);
2614     return ret;
2615 }
2616
2617
2618 int
2619 qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
2620                            virConnectPtr dconn,
2621                            const char *cookiein,
2622                            int cookieinlen,
2623                            char **cookieout,
2624                            int *cookieoutlen,
2625                            const char *uri_in,
2626                            char **uri_out,
2627                            virDomainDefPtr *def,
2628                            const char *origname,
2629                            const char *listenAddress,
2630                            unsigned long flags)
2631 {
2632     unsigned short port = 0;
2633     bool autoPort = true;
2634     char *hostname = NULL;
2635     const char *p;
2636     char *uri_str = NULL;
2637     int ret = -1;
2638     virURIPtr uri = NULL;
2639     bool well_formed_uri = true;
2640     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
2641     const char *migrateHost = cfg->migrateHost;
2642
2643     VIR_DEBUG("driver=%p, dconn=%p, cookiein=%s, cookieinlen=%d, "
2644               "cookieout=%p, cookieoutlen=%p, uri_in=%s, uri_out=%p, "
2645               "def=%p, origname=%s, listenAddress=%s, flags=%lx",
2646               driver, dconn, NULLSTR(cookiein), cookieinlen,
2647               cookieout, cookieoutlen, NULLSTR(uri_in), uri_out,
2648               *def, origname, NULLSTR(listenAddress), flags);
2649
2650     *uri_out = NULL;
2651
2652     /* The URI passed in may be NULL or a string "tcp://somehostname:port".
2653      *
2654      * If the URI passed in is NULL then we allocate a port number
2655      * from our pool of port numbers, and if the migrateHost is configured,
2656      * we return a URI of "tcp://migrateHost:port", otherwise return a URI
2657      * of "tcp://ourhostname:port".
2658      *
2659      * If the URI passed in is not NULL then we try to parse out the
2660      * port number and use that (note that the hostname is assumed
2661      * to be a correct hostname which refers to the target machine).
2662      */
2663     if (uri_in == NULL) {
2664         if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0)
2665             goto cleanup;
2666
2667         if (migrateHost != NULL) {
2668             if (virSocketAddrIsNumeric(migrateHost) &&
2669                 virSocketAddrParse(NULL, migrateHost, AF_UNSPEC) < 0)
2670                 goto cleanup;
2671
2672            if (VIR_STRDUP(hostname, migrateHost) < 0)
2673                 goto cleanup;
2674         } else {
2675             if ((hostname = virGetHostname()) == NULL)
2676                 goto cleanup;
2677         }
2678
2679         if (STRPREFIX(hostname, "localhost")) {
2680             virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
2681                            _("hostname on destination resolved to localhost,"
2682                              " but migration requires an FQDN"));
2683             goto cleanup;
2684         }
2685
2686         /* XXX this really should have been a properly well-formed
2687          * URI, but we can't add in tcp:// now without breaking
2688          * compatibility with old targets. We at least make the
2689          * new targets accept both syntaxes though.
2690          */
2691         /* Caller frees */
2692         if (virAsprintf(uri_out, "tcp:%s:%d", hostname, port) < 0)
2693             goto cleanup;
2694     } else {
2695         /* Check the URI starts with "tcp:".  We will escape the
2696          * URI when passing it to the qemu monitor, so bad
2697          * characters in hostname part don't matter.
2698          */
2699         if (!(p = STRSKIP(uri_in, "tcp:"))) {
2700             virReportError(VIR_ERR_INVALID_ARG, "%s",
2701                            _("only tcp URIs are supported for KVM/QEMU"
2702                              " migrations"));
2703             goto cleanup;
2704         }
2705
2706         /* Convert uri_in to well-formed URI with // after tcp: */
2707         if (!(STRPREFIX(uri_in, "tcp://"))) {
2708             well_formed_uri = false;
2709             if (virAsprintf(&uri_str, "tcp://%s", p) < 0)
2710                 goto cleanup;
2711         }
2712
2713         uri = virURIParse(uri_str ? uri_str : uri_in);
2714         VIR_FREE(uri_str);
2715
2716         if (uri == NULL) {
2717             virReportError(VIR_ERR_INVALID_ARG, _("unable to parse URI: %s"),
2718                            uri_in);
2719             goto cleanup;
2720         }
2721
2722         if (uri->server == NULL) {
2723             virReportError(VIR_ERR_INVALID_ARG, _("missing host in migration"
2724                                                   " URI: %s"), uri_in);
2725             goto cleanup;
2726         }
2727
2728         if (uri->port == 0) {
2729             if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0)
2730                 goto cleanup;
2731
2732             if (well_formed_uri) {
2733                 uri->port = port;
2734
2735                 /* Caller frees */
2736                 if (!(*uri_out = virURIFormat(uri)))
2737                     goto cleanup;
2738             } else {
2739                 /* Caller frees */
2740                 if (virAsprintf(uri_out, "%s:%d", uri_in, port) < 0)
2741                     goto cleanup;
2742             }
2743
2744         } else {
2745             port = uri->port;
2746             autoPort = false;
2747         }
2748     }
2749
2750     if (*uri_out)
2751         VIR_DEBUG("Generated uri_out=%s", *uri_out);
2752
2753     ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen,
2754                                   cookieout, cookieoutlen, def, origname,
2755                                   NULL, port, autoPort, listenAddress, flags);
2756  cleanup:
2757     virURIFree(uri);
2758     VIR_FREE(hostname);
2759     virObjectUnref(cfg);
2760     if (ret != 0) {
2761         VIR_FREE(*uri_out);
2762         if (autoPort)
2763             virPortAllocatorRelease(driver->migrationPorts, port);
2764     }
2765     return ret;
2766 }
2767
2768
2769 virDomainDefPtr
2770 qemuMigrationPrepareDef(virQEMUDriverPtr driver,
2771                         const char *dom_xml,
2772                         const char *dname,
2773                         char **origname)
2774 {
2775     virCapsPtr caps = NULL;
2776     virDomainDefPtr def;
2777     char *name = NULL;
2778
2779     if (!dom_xml) {
2780         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
2781                        _("no domain XML passed"));
2782         return NULL;
2783     }
2784
2785     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
2786         return NULL;
2787
2788     if (!(def = virDomainDefParseString(dom_xml, caps, driver->xmlopt,
2789                                         QEMU_EXPECTED_VIRT_TYPES,
2790                                         VIR_DOMAIN_XML_INACTIVE)))
2791         goto cleanup;
2792
2793     if (dname) {
2794         name = def->name;
2795         if (VIR_STRDUP(def->name, dname) < 0) {
2796             virDomainDefFree(def);
2797             def = NULL;
2798         }
2799     }
2800
2801  cleanup:
2802     virObjectUnref(caps);
2803     if (def && origname)
2804         *origname = name;
2805     else
2806         VIR_FREE(name);
2807     return def;
2808 }
2809
2810
2811 static int
2812 qemuMigrationConfirmPhase(virQEMUDriverPtr driver,
2813                           virConnectPtr conn,
2814                           virDomainObjPtr vm,
2815                           const char *cookiein,
2816                           int cookieinlen,
2817                           unsigned int flags,
2818                           int retcode)
2819 {
2820     qemuMigrationCookiePtr mig;
2821     virObjectEventPtr event = NULL;
2822     int rv = -1;
2823     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
2824
2825     VIR_DEBUG("driver=%p, conn=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
2826               "flags=%x, retcode=%d",
2827               driver, conn, vm, NULLSTR(cookiein), cookieinlen,
2828               flags, retcode);
2829
2830     virCheckFlags(QEMU_MIGRATION_FLAGS, -1);
2831
2832     qemuMigrationJobSetPhase(driver, vm,
2833                              retcode == 0
2834                              ? QEMU_MIGRATION_PHASE_CONFIRM3
2835                              : QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED);
2836
2837     if (!(mig = qemuMigrationEatCookie(driver, vm, cookiein, cookieinlen, 0)))
2838         goto cleanup;
2839
2840     if (flags & VIR_MIGRATE_OFFLINE)
2841         goto done;
2842
2843     /* Did the migration go as planned?  If yes, kill off the
2844      * domain object, but if no, resume CPUs
2845      */
2846     if (retcode == 0) {
2847         /* If guest uses SPICE and supports seamless migration we have to hold
2848          * up domain shutdown until SPICE server transfers its data */
2849         qemuMigrationWaitForSpice(driver, vm);
2850
2851         qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_MIGRATED,
2852                         VIR_QEMU_PROCESS_STOP_MIGRATED);
2853         virDomainAuditStop(vm, "migrated");
2854
2855         event = virDomainEventLifecycleNewFromObj(vm,
2856                                          VIR_DOMAIN_EVENT_STOPPED,
2857                                          VIR_DOMAIN_EVENT_STOPPED_MIGRATED);
2858     } else {
2859
2860         /* cancel any outstanding NBD jobs */
2861         qemuMigrationCancelDriveMirror(mig, driver, vm);
2862
2863         if (qemuMigrationRestoreDomainState(conn, vm)) {
2864             event = virDomainEventLifecycleNewFromObj(vm,
2865                                                       VIR_DOMAIN_EVENT_RESUMED,
2866                                                       VIR_DOMAIN_EVENT_RESUMED_MIGRATED);
2867         }
2868
2869         if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0) {
2870             VIR_WARN("Failed to save status on vm %s", vm->def->name);
2871             goto cleanup;
2872         }
2873     }
2874
2875  done:
2876     qemuMigrationCookieFree(mig);
2877     rv = 0;
2878
2879  cleanup:
2880     if (event)
2881         qemuDomainEventQueue(driver, event);
2882     virObjectUnref(cfg);
2883     return rv;
2884 }
2885
2886 int
2887 qemuMigrationConfirm(virConnectPtr conn,
2888                      virDomainObjPtr vm,
2889                      const char *cookiein,
2890                      int cookieinlen,
2891                      unsigned int flags,
2892                      int cancelled)
2893 {
2894     virQEMUDriverPtr driver = conn->privateData;
2895     enum qemuMigrationJobPhase phase;
2896     virQEMUDriverConfigPtr cfg = NULL;
2897     int ret = -1;
2898
2899     cfg = virQEMUDriverGetConfig(driver);
2900
2901     if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
2902         goto cleanup;
2903
2904     if (cancelled)
2905         phase = QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED;
2906     else
2907         phase = QEMU_MIGRATION_PHASE_CONFIRM3;
2908
2909     qemuMigrationJobStartPhase(driver, vm, phase);
2910     virCloseCallbacksUnset(driver->closeCallbacks, vm,
2911                            qemuMigrationCleanup);
2912
2913     ret = qemuMigrationConfirmPhase(driver, conn, vm,
2914                                     cookiein, cookieinlen,
2915                                     flags, cancelled);
2916
2917     if (qemuMigrationJobFinish(driver, vm) == 0) {
2918         vm = NULL;
2919     } else if (!virDomainObjIsActive(vm) &&
2920                (!vm->persistent || (flags & VIR_MIGRATE_UNDEFINE_SOURCE))) {
2921         if (flags & VIR_MIGRATE_UNDEFINE_SOURCE)
2922             virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm);
2923         qemuDomainRemoveInactive(driver, vm);
2924         vm = NULL;
2925     }
2926
2927  cleanup:
2928     if (vm)
2929         virObjectUnlock(vm);
2930     virObjectUnref(cfg);
2931     return ret;
2932 }
2933
2934
2935 enum qemuMigrationDestinationType {
2936     MIGRATION_DEST_HOST,
2937     MIGRATION_DEST_CONNECT_HOST,
2938     MIGRATION_DEST_UNIX,
2939     MIGRATION_DEST_FD,
2940 };
2941
2942 enum qemuMigrationForwardType {
2943     MIGRATION_FWD_DIRECT,
2944     MIGRATION_FWD_STREAM,
2945 };
2946
2947 typedef struct _qemuMigrationSpec qemuMigrationSpec;
2948 typedef qemuMigrationSpec *qemuMigrationSpecPtr;
2949 struct _qemuMigrationSpec {
2950     enum qemuMigrationDestinationType destType;
2951     union {
2952         struct {
2953             const char *name;
2954             int port;
2955         } host;
2956
2957         struct {
2958             char *file;
2959             int sock;
2960         } unix_socket;
2961
2962         struct {
2963             int qemu;
2964             int local;
2965         } fd;
2966     } dest;
2967
2968     enum qemuMigrationForwardType fwdType;
2969     union {
2970         virStreamPtr stream;
2971     } fwd;
2972 };
2973
2974 #define TUNNEL_SEND_BUF_SIZE 65536
2975
2976 typedef struct _qemuMigrationIOThread qemuMigrationIOThread;
2977 typedef qemuMigrationIOThread *qemuMigrationIOThreadPtr;
2978 struct _qemuMigrationIOThread {
2979     virThread thread;
2980     virStreamPtr st;
2981     int sock;
2982     virError err;
2983     int wakeupRecvFD;
2984     int wakeupSendFD;
2985 };
2986
2987 static void qemuMigrationIOFunc(void *arg)
2988 {
2989     qemuMigrationIOThreadPtr data = arg;
2990     char *buffer = NULL;
2991     struct pollfd fds[2];
2992     int timeout = -1;
2993     virErrorPtr err = NULL;
2994
2995     VIR_DEBUG("Running migration tunnel; stream=%p, sock=%d",
2996               data->st, data->sock);
2997
2998     if (VIR_ALLOC_N(buffer, TUNNEL_SEND_BUF_SIZE) < 0)
2999         goto abrt;
3000
3001     fds[0].fd = data->sock;
3002     fds[1].fd = data->wakeupRecvFD;
3003
3004     for (;;) {
3005         int ret;
3006
3007         fds[0].events = fds[1].events = POLLIN;
3008         fds[0].revents = fds[1].revents = 0;
3009
3010         ret = poll(fds, ARRAY_CARDINALITY(fds), timeout);
3011
3012         if (ret < 0) {
3013             if (errno == EAGAIN || errno == EINTR)
3014                 continue;
3015             virReportSystemError(errno, "%s",
3016                                  _("poll failed in migration tunnel"));
3017             goto abrt;
3018         }
3019
3020         if (ret == 0) {
3021             /* We were asked to gracefully stop but reading would block. This
3022              * can only happen if qemu told us migration finished but didn't
3023              * close the migration fd. We handle this in the same way as EOF.
3024              */
3025             VIR_DEBUG("QEMU forgot to close migration fd");
3026             break;
3027         }
3028
3029         if (fds[1].revents & (POLLIN | POLLERR | POLLHUP)) {
3030             char stop = 0;
3031
3032             if (saferead(data->wakeupRecvFD, &stop, 1) != 1) {
3033                 virReportSystemError(errno, "%s",
3034                                      _("failed to read from wakeup fd"));
3035                 goto abrt;
3036             }
3037
3038             VIR_DEBUG("Migration tunnel was asked to %s",
3039                       stop ? "abort" : "finish");
3040             if (stop) {
3041                 goto abrt;
3042             } else {
3043                 timeout = 0;
3044             }
3045         }
3046
3047         if (fds[0].revents & (POLLIN | POLLERR | POLLHUP)) {
3048             int nbytes;
3049
3050             nbytes = saferead(data->sock, buffer, TUNNEL_SEND_BUF_SIZE);
3051             if (nbytes > 0) {
3052                 if (virStreamSend(data->st, buffer, nbytes) < 0)
3053                     goto error;
3054             } else if (nbytes < 0) {
3055                 virReportSystemError(errno, "%s",
3056                         _("tunnelled migration failed to read from qemu"));
3057                 goto abrt;
3058             } else {
3059                 /* EOF; get out of here */
3060                 break;
3061             }
3062         }
3063     }
3064
3065     if (virStreamFinish(data->st) < 0)
3066         goto error;
3067
3068     VIR_FREE(buffer);
3069
3070     return;
3071
3072  abrt:
3073     err = virSaveLastError();
3074     if (err && err->code == VIR_ERR_OK) {
3075         virFreeError(err);
3076         err = NULL;
3077     }
3078     virStreamAbort(data->st);
3079     if (err) {
3080         virSetError(err);
3081         virFreeError(err);
3082     }
3083
3084  error:
3085     virCopyLastError(&data->err);
3086     virResetLastError();
3087     VIR_FREE(buffer);
3088 }
3089
3090
3091 static qemuMigrationIOThreadPtr
3092 qemuMigrationStartTunnel(virStreamPtr st,
3093                          int sock)
3094 {
3095     qemuMigrationIOThreadPtr io = NULL;
3096     int wakeupFD[2] = { -1, -1 };
3097
3098     if (pipe2(wakeupFD, O_CLOEXEC) < 0) {
3099         virReportSystemError(errno, "%s",
3100                              _("Unable to make pipe"));
3101         goto error;
3102     }
3103
3104     if (VIR_ALLOC(io) < 0)
3105         goto error;
3106
3107     io->st = st;
3108     io->sock = sock;
3109     io->wakeupRecvFD = wakeupFD[0];
3110     io->wakeupSendFD = wakeupFD[1];
3111
3112     if (virThreadCreate(&io->thread, true,
3113                         qemuMigrationIOFunc,
3114                         io) < 0) {
3115         virReportSystemError(errno, "%s",
3116                              _("Unable to create migration thread"));
3117         goto error;
3118     }
3119
3120     return io;
3121
3122  error:
3123     VIR_FORCE_CLOSE(wakeupFD[0]);
3124     VIR_FORCE_CLOSE(wakeupFD[1]);
3125     VIR_FREE(io);
3126     return NULL;
3127 }
3128
3129 static int
3130 qemuMigrationStopTunnel(qemuMigrationIOThreadPtr io, bool error)
3131 {
3132     int rv = -1;
3133     char stop = error ? 1 : 0;
3134
3135     /* make sure the thread finishes its job and is joinable */
3136     if (safewrite(io->wakeupSendFD, &stop, 1) != 1) {
3137         virReportSystemError(errno, "%s",
3138                              _("failed to wakeup migration tunnel"));
3139         goto cleanup;
3140     }
3141
3142     virThreadJoin(&io->thread);
3143
3144     /* Forward error from the IO thread, to this thread */
3145     if (io->err.code != VIR_ERR_OK) {
3146         if (error)
3147             rv = 0;
3148         else
3149             virSetError(&io->err);
3150         virResetError(&io->err);
3151         goto cleanup;
3152     }
3153
3154     rv = 0;
3155
3156  cleanup:
3157     VIR_FORCE_CLOSE(io->wakeupSendFD);
3158     VIR_FORCE_CLOSE(io->wakeupRecvFD);
3159     VIR_FREE(io);
3160     return rv;
3161 }
3162
3163 static int
3164 qemuMigrationConnect(virQEMUDriverPtr driver,
3165                      virDomainObjPtr vm,
3166                      qemuMigrationSpecPtr spec)
3167 {
3168     virNetSocketPtr sock;
3169     const char *host;
3170     char *port = NULL;
3171     int ret = -1;
3172
3173     host = spec->dest.host.name;
3174     if (virAsprintf(&port, "%d", spec->dest.host.port) < 0)
3175         return -1;
3176
3177     spec->destType = MIGRATION_DEST_FD;
3178     spec->dest.fd.qemu = -1;
3179
3180     if (virSecurityManagerSetSocketLabel(driver->securityManager, vm->def) < 0)
3181         goto cleanup;
3182     if (virNetSocketNewConnectTCP(host, port, &sock) == 0) {
3183         spec->dest.fd.qemu = virNetSocketDupFD(sock, true);
3184         virObjectUnref(sock);
3185     }
3186     if (virSecurityManagerClearSocketLabel(driver->securityManager, vm->def) < 0 ||
3187         spec->dest.fd.qemu == -1)
3188         goto cleanup;
3189
3190     /* Migration expects a blocking FD */
3191     if (virSetBlocking(spec->dest.fd.qemu, true) < 0) {
3192         virReportSystemError(errno, _("Unable to set FD %d blocking"),
3193                              spec->dest.fd.qemu);
3194         goto cleanup;
3195     }
3196
3197     ret = 0;
3198
3199  cleanup:
3200     VIR_FREE(port);
3201     if (ret < 0)
3202         VIR_FORCE_CLOSE(spec->dest.fd.qemu);
3203     return ret;
3204 }
3205
3206 static int
3207 qemuMigrationRun(virQEMUDriverPtr driver,
3208                  virDomainObjPtr vm,
3209                  const char *cookiein,
3210                  int cookieinlen,
3211                  char **cookieout,
3212                  int *cookieoutlen,
3213                  unsigned long flags,
3214                  unsigned long resource,
3215                  qemuMigrationSpecPtr spec,
3216                  virConnectPtr dconn,
3217                  const char *graphicsuri)
3218 {
3219     int ret = -1;
3220     unsigned int migrate_flags = QEMU_MONITOR_MIGRATE_BACKGROUND;
3221     qemuDomainObjPrivatePtr priv = vm->privateData;
3222     qemuMigrationCookiePtr mig = NULL;
3223     qemuMigrationIOThreadPtr iothread = NULL;
3224     int fd = -1;
3225     unsigned long migrate_speed = resource ? resource : priv->migMaxBandwidth;
3226     virErrorPtr orig_err = NULL;
3227     unsigned int cookieFlags = 0;
3228     bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR);
3229     int rc;
3230
3231     VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
3232               "cookieout=%p, cookieoutlen=%p, flags=%lx, resource=%lu, "
3233               "spec=%p (dest=%d, fwd=%d), dconn=%p, graphicsuri=%s",
3234               driver, vm, NULLSTR(cookiein), cookieinlen,
3235               cookieout, cookieoutlen, flags, resource,
3236               spec, spec->destType, spec->fwdType, dconn,
3237               NULLSTR(graphicsuri));
3238
3239     if (flags & VIR_MIGRATE_NON_SHARED_DISK) {
3240         migrate_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK;
3241         cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
3242     }
3243
3244     if (flags & VIR_MIGRATE_NON_SHARED_INC) {
3245         migrate_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC;
3246         cookieFlags |= QEMU_MIGRATION_COOKIE_NBD;
3247     }
3248
3249     if (virLockManagerPluginUsesState(driver->lockManager) &&
3250         !cookieout) {
3251         virReportError(VIR_ERR_INTERNAL_ERROR,
3252                        _("Migration with lock driver %s requires"
3253                          " cookie support"),
3254                        virLockManagerPluginGetName(driver->lockManager));
3255         return -1;
3256     }
3257
3258     mig = qemuMigrationEatCookie(driver, vm, cookiein, cookieinlen,
3259                                  cookieFlags | QEMU_MIGRATION_COOKIE_GRAPHICS);
3260     if (!mig)
3261         goto cleanup;
3262
3263     if (qemuDomainMigrateGraphicsRelocate(driver, vm, mig, graphicsuri) < 0)
3264         VIR_WARN("unable to provide data for graphics client relocation");
3265
3266     /* this will update migrate_flags on success */
3267     if (qemuMigrationDriveMirror(driver, vm, mig, spec->dest.host.name,
3268                                  migrate_speed, &migrate_flags) < 0) {
3269         /* error reported by helper func */
3270         goto cleanup;
3271     }
3272
3273     /* Before EnterMonitor, since qemuMigrationSetOffline already does that */
3274     if (!(flags & VIR_MIGRATE_LIVE) &&
3275         virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) {
3276         if (qemuMigrationSetOffline(driver, vm) < 0)
3277             goto cleanup;
3278     }
3279
3280     if (flags & VIR_MIGRATE_COMPRESSED &&
3281         qemuMigrationSetCompression(driver, vm,
3282                                     QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
3283         goto cleanup;
3284
3285     if (flags & VIR_MIGRATE_AUTO_CONVERGE &&
3286         qemuMigrationSetAutoConverge(driver, vm,
3287                                      QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
3288         goto cleanup;
3289
3290     if (qemuDomainObjEnterMonitorAsync(driver, vm,
3291                                        QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
3292         goto cleanup;
3293
3294     if (priv->job.asyncAbort) {
3295         /* explicitly do this *after* we entered the monitor,
3296          * as this is a critical section so we are guaranteed
3297          * priv->job.asyncAbort will not change */
3298         qemuDomainObjExitMonitor(driver, vm);
3299         virReportError(VIR_ERR_OPERATION_ABORTED, _("%s: %s"),
3300                        qemuDomainAsyncJobTypeToString(priv->job.asyncJob),
3301                        _("canceled by client"));
3302         goto cleanup;
3303     }
3304
3305     if (qemuMonitorSetMigrationSpeed(priv->mon, migrate_speed) < 0) {
3306         qemuDomainObjExitMonitor(driver, vm);
3307         goto cleanup;
3308     }
3309
3310     /* connect to the destination qemu if needed */
3311     if (spec->destType == MIGRATION_DEST_CONNECT_HOST &&
3312         qemuMigrationConnect(driver, vm, spec) < 0) {
3313         qemuDomainObjExitMonitor(driver, vm);
3314         goto cleanup;
3315     }
3316
3317     switch (spec->destType) {
3318     case MIGRATION_DEST_HOST:
3319         ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
3320                                        spec->dest.host.name,
3321                                        spec->dest.host.port);
3322         break;
3323
3324     case MIGRATION_DEST_CONNECT_HOST:
3325         /* handled above and transformed into MIGRATION_DEST_FD */
3326         break;
3327
3328     case MIGRATION_DEST_UNIX:
3329         if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_UNIX)) {
3330             ret = qemuMonitorMigrateToUnix(priv->mon, migrate_flags,
3331                                            spec->dest.unix_socket.file);
3332         } else {
3333             const char *args[] = {
3334                 "nc", "-U", spec->dest.unix_socket.file, NULL
3335             };
3336             ret = qemuMonitorMigrateToCommand(priv->mon, migrate_flags, args);
3337         }
3338         break;
3339
3340     case MIGRATION_DEST_FD:
3341         if (spec->fwdType != MIGRATION_FWD_DIRECT) {
3342             fd = spec->dest.fd.local;
3343             spec->dest.fd.local = -1;
3344         }
3345         ret = qemuMonitorMigrateToFd(priv->mon, migrate_flags,
3346                                      spec->dest.fd.qemu);
3347         VIR_FORCE_CLOSE(spec->dest.fd.qemu);
3348         break;
3349     }
3350     qemuDomainObjExitMonitor(driver, vm);
3351     if (ret < 0)
3352         goto cleanup;
3353     ret = -1;
3354
3355     if (!virDomainObjIsActive(vm)) {
3356         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3357                        _("guest unexpectedly quit"));
3358         goto cleanup;
3359     }
3360
3361     /* From this point onwards we *must* call cancel to abort the
3362      * migration on source if anything goes wrong */
3363
3364     if (spec->destType == MIGRATION_DEST_UNIX) {
3365         /* It is also possible that the migrate didn't fail initially, but
3366          * rather failed later on.  Check its status before waiting for a
3367          * connection from qemu which may never be initiated.
3368          */
3369         if (qemuMigrationUpdateJobStatus(driver, vm, _("migration job"),
3370                                          QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
3371             goto cancel;
3372
3373         while ((fd = accept(spec->dest.unix_socket.sock, NULL, NULL)) < 0) {
3374             if (errno == EAGAIN || errno == EINTR)
3375                 continue;
3376             virReportSystemError(errno, "%s",
3377                                  _("failed to accept connection from qemu"));
3378             goto cancel;
3379         }
3380     }
3381
3382     if (spec->fwdType != MIGRATION_FWD_DIRECT &&
3383         !(iothread = qemuMigrationStartTunnel(spec->fwd.stream, fd)))
3384         goto cancel;
3385
3386     rc = qemuMigrationWaitForCompletion(driver, vm,
3387                                         QEMU_ASYNC_JOB_MIGRATION_OUT,
3388                                         dconn, abort_on_error);
3389     if (rc == -2)
3390         goto cancel;
3391     else if (rc == -1)
3392         goto cleanup;
3393
3394     /* When migration completed, QEMU will have paused the
3395      * CPUs for us, but unless we're using the JSON monitor
3396      * we won't have been notified of this, so might still
3397      * think we're running. For v2 protocol this doesn't
3398      * matter because we'll kill the VM soon, but for v3
3399      * this is important because we stay paused until the
3400      * confirm3 step, but need to release the lock state
3401      */
3402     if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) {
3403         if (qemuMigrationSetOffline(driver, vm) < 0)
3404             goto cleanup;
3405     }
3406
3407     ret = 0;
3408
3409  cleanup:
3410     if (ret < 0 && !orig_err)
3411         orig_err = virSaveLastError();
3412
3413     /* cancel any outstanding NBD jobs */
3414     qemuMigrationCancelDriveMirror(mig, driver, vm);
3415
3416     if (spec->fwdType != MIGRATION_FWD_DIRECT) {
3417         if (iothread && qemuMigrationStopTunnel(iothread, ret < 0) < 0)
3418             ret = -1;
3419         VIR_FORCE_CLOSE(fd);
3420     }
3421
3422     cookieFlags |= QEMU_MIGRATION_COOKIE_NETWORK;
3423     if (flags & VIR_MIGRATE_PERSIST_DEST)
3424         cookieFlags |= QEMU_MIGRATION_COOKIE_PERSISTENT;
3425     if (ret == 0 &&
3426         qemuMigrationBakeCookie(mig, driver, vm, cookieout,
3427                                 cookieoutlen, cookieFlags) < 0) {
3428         VIR_WARN("Unable to encode migration cookie");
3429     }
3430
3431     qemuMigrationCookieFree(mig);
3432
3433     if (orig_err) {
3434         virSetError(orig_err);
3435         virFreeError(orig_err);
3436     }
3437
3438     return ret;
3439
3440  cancel:
3441     orig_err = virSaveLastError();
3442
3443     if (virDomainObjIsActive(vm)) {
3444         if (qemuDomainObjEnterMonitorAsync(driver, vm,
3445                                            QEMU_ASYNC_JOB_MIGRATION_OUT) == 0) {
3446             qemuMonitorMigrateCancel(priv->mon);
3447             qemuDomainObjExitMonitor(driver, vm);
3448         }
3449     }
3450     goto cleanup;
3451 }
3452
3453 /* Perform migration using QEMU's native TCP migrate support,
3454  * not encrypted obviously
3455  */
3456 static int doNativeMigrate(virQEMUDriverPtr driver,
3457                            virDomainObjPtr vm,
3458                            const char *uri,
3459                            const char *cookiein,
3460                            int cookieinlen,
3461                            char **cookieout,
3462                            int *cookieoutlen,
3463                            unsigned long flags,
3464                            unsigned long resource,
3465                            virConnectPtr dconn,
3466                            const char *graphicsuri)
3467 {
3468     qemuDomainObjPrivatePtr priv = vm->privateData;
3469     virURIPtr uribits = NULL;
3470     int ret = -1;
3471     qemuMigrationSpec spec;
3472
3473     VIR_DEBUG("driver=%p, vm=%p, uri=%s, cookiein=%s, cookieinlen=%d, "
3474               "cookieout=%p, cookieoutlen=%p, flags=%lx, resource=%lu, "
3475               "graphicsuri=%s",
3476               driver, vm, uri, NULLSTR(cookiein), cookieinlen,
3477               cookieout, cookieoutlen, flags, resource,
3478               NULLSTR(graphicsuri));
3479
3480     if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri, "tcp://")) {
3481         char *tmp;
3482         /* HACK: source host generates bogus URIs, so fix them up */
3483         if (virAsprintf(&tmp, "tcp://%s", uri + strlen("tcp:")) < 0)
3484             return -1;
3485         uribits = virURIParse(tmp);
3486         VIR_FREE(tmp);
3487     } else {
3488         uribits = virURIParse(uri);
3489     }
3490     if (!uribits)
3491         return -1;
3492
3493     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))
3494         spec.destType = MIGRATION_DEST_CONNECT_HOST;
3495     else
3496         spec.destType = MIGRATION_DEST_HOST;
3497     spec.dest.host.name = uribits->server;
3498     spec.dest.host.port = uribits->port;
3499     spec.fwdType = MIGRATION_FWD_DIRECT;
3500
3501     ret = qemuMigrationRun(driver, vm, cookiein, cookieinlen, cookieout,
3502                            cookieoutlen, flags, resource, &spec, dconn,
3503                            graphicsuri);
3504
3505     if (spec.destType == MIGRATION_DEST_FD)
3506         VIR_FORCE_CLOSE(spec.dest.fd.qemu);
3507
3508     virURIFree(uribits);
3509
3510     return ret;
3511 }
3512
3513
3514 static int doTunnelMigrate(virQEMUDriverPtr driver,
3515                            virDomainObjPtr vm,
3516                            virStreamPtr st,
3517                            const char *cookiein,
3518                            int cookieinlen,
3519                            char **cookieout,
3520                            int *cookieoutlen,
3521                            unsigned long flags,
3522                            unsigned long resource,
3523                            virConnectPtr dconn,
3524                            const char *graphicsuri)
3525 {
3526     qemuDomainObjPrivatePtr priv = vm->privateData;
3527     virNetSocketPtr sock = NULL;
3528     int ret = -1;
3529     qemuMigrationSpec spec;
3530     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
3531
3532     VIR_DEBUG("driver=%p, vm=%p, st=%p, cookiein=%s, cookieinlen=%d, "
3533               "cookieout=%p, cookieoutlen=%p, flags=%lx, resource=%lu, "
3534               "graphicsuri=%s",
3535               driver, vm, st, NULLSTR(cookiein), cookieinlen,
3536               cookieout, cookieoutlen, flags, resource,
3537               NULLSTR(graphicsuri));
3538
3539     if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
3540         !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_UNIX) &&
3541         !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_EXEC)) {
3542         virReportError(VIR_ERR_OPERATION_FAILED, "%s",
3543                        _("Source qemu is too old to support tunnelled migration"));
3544         virObjectUnref(cfg);
3545         return -1;
3546     }
3547
3548     spec.fwdType = MIGRATION_FWD_STREAM;
3549     spec.fwd.stream = st;
3550
3551     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {
3552         int fds[2];
3553
3554         spec.destType = MIGRATION_DEST_FD;
3555         spec.dest.fd.qemu = -1;
3556         spec.dest.fd.local = -1;
3557
3558         if (pipe2(fds, O_CLOEXEC) == 0) {
3559             spec.dest.fd.qemu = fds[1];
3560             spec.dest.fd.local = fds[0];
3561         }
3562         if (spec.dest.fd.qemu == -1 ||
3563             virSecurityManagerSetImageFDLabel(driver->securityManager, vm->def,
3564                                               spec.dest.fd.qemu) < 0) {
3565             virReportSystemError(errno, "%s",
3566                         _("cannot create pipe for tunnelled migration"));
3567             goto cleanup;
3568         }
3569     } else {
3570         spec.destType = MIGRATION_DEST_UNIX;
3571         spec.dest.unix_socket.sock = -1;
3572         spec.dest.unix_socket.file = NULL;
3573
3574         if (virAsprintf(&spec.dest.unix_socket.file,
3575                         "%s/qemu.tunnelmigrate.src.%s",
3576                         cfg->libDir, vm->def->name) < 0)
3577             goto cleanup;
3578
3579         if (virNetSocketNewListenUNIX(spec.dest.unix_socket.file, 0700,
3580                                       cfg->user, cfg->group,
3581                                       &sock) < 0 ||
3582             virNetSocketListen(sock, 1) < 0)
3583             goto cleanup;
3584
3585         spec.dest.unix_socket.sock = virNetSocketGetFD(sock);
3586     }
3587
3588     ret = qemuMigrationRun(driver, vm, cookiein, cookieinlen, cookieout,
3589                            cookieoutlen, flags, resource, &spec, dconn,
3590                            graphicsuri);
3591
3592  cleanup:
3593     if (spec.destType == MIGRATION_DEST_FD) {
3594         VIR_FORCE_CLOSE(spec.dest.fd.qemu);
3595         VIR_FORCE_CLOSE(spec.dest.fd.local);
3596     } else {
3597         virObjectUnref(sock);
3598         VIR_FREE(spec.dest.unix_socket.file);
3599     }
3600
3601     virObjectUnref(cfg);
3602     return ret;
3603 }
3604
3605
3606 /* This is essentially a re-impl of virDomainMigrateVersion2
3607  * from libvirt.c, but running in source libvirtd context,
3608  * instead of client app context & also adding in tunnel
3609  * handling */
3610 static int doPeer2PeerMigrate2(virQEMUDriverPtr driver,
3611                                virConnectPtr sconn ATTRIBUTE_UNUSED,
3612                                virConnectPtr dconn,
3613                                virDomainObjPtr vm,
3614                                const char *dconnuri,
3615                                unsigned long flags,
3616                                const char *dname,
3617                                unsigned long resource)
3618 {
3619     virDomainPtr ddomain = NULL;
3620     char *uri_out = NULL;
3621     char *cookie = NULL;
3622     char *dom_xml = NULL;
3623     int cookielen = 0, ret;
3624     virErrorPtr orig_err = NULL;
3625     bool cancelled;
3626     virStreamPtr st = NULL;
3627     unsigned long destflags;
3628
3629     VIR_DEBUG("driver=%p, sconn=%p, dconn=%p, vm=%p, dconnuri=%s, "
3630               "flags=%lx, dname=%s, resource=%lu",
3631               driver, sconn, dconn, vm, NULLSTR(dconnuri),
3632               flags, NULLSTR(dname), resource);
3633
3634     /* In version 2 of the protocol, the prepare step is slightly
3635      * different.  We fetch the domain XML of the source domain
3636      * and pass it to Prepare2.
3637      */
3638     if (!(dom_xml = qemuDomainFormatXML(driver, vm,
3639                                         QEMU_DOMAIN_FORMAT_LIVE_FLAGS |
3640                                         VIR_DOMAIN_XML_MIGRATABLE)))
3641         return -1;
3642
3643     if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED)
3644         flags |= VIR_MIGRATE_PAUSED;
3645
3646     destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR |
3647                           VIR_MIGRATE_AUTO_CONVERGE);
3648
3649     VIR_DEBUG("Prepare2 %p", dconn);
3650     if (flags & VIR_MIGRATE_TUNNELLED) {
3651         /*
3652          * Tunnelled Migrate Version 2 does not support cookies
3653          * due to missing parameters in the prepareTunnel() API.
3654          */
3655
3656         if (!(st = virStreamNew(dconn, 0)))
3657             goto cleanup;
3658
3659         qemuDomainObjEnterRemote(vm);
3660         ret = dconn->driver->domainMigratePrepareTunnel
3661             (dconn, st, destflags, dname, resource, dom_xml);
3662         qemuDomainObjExitRemote(vm);
3663     } else {
3664         qemuDomainObjEnterRemote(vm);
3665         ret = dconn->driver->domainMigratePrepare2
3666             (dconn, &cookie, &cookielen, NULL, &uri_out,
3667              destflags, dname, resource, dom_xml);
3668         qemuDomainObjExitRemote(vm);
3669     }
3670     VIR_FREE(dom_xml);
3671     if (ret == -1)
3672         goto cleanup;
3673
3674     /* the domain may have shutdown or crashed while we had the locks dropped
3675      * in qemuDomainObjEnterRemote, so check again
3676      */
3677     if (!virDomainObjIsActive(vm)) {
3678         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3679                        _("guest unexpectedly quit"));
3680         goto cleanup;
3681     }
3682
3683     if (!(flags & VIR_MIGRATE_TUNNELLED) &&
3684         (uri_out == NULL)) {
3685         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3686                        _("domainMigratePrepare2 did not set uri"));
3687         cancelled = true;
3688         orig_err = virSaveLastError();
3689         goto finish;
3690     }
3691
3692     /* Perform the migration.  The driver isn't supposed to return
3693      * until the migration is complete.
3694      */
3695     VIR_DEBUG("Perform %p", sconn);
3696     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM2);
3697     if (flags & VIR_MIGRATE_TUNNELLED)
3698         ret = doTunnelMigrate(driver, vm, st,
3699                               NULL, 0, NULL, NULL,
3700                               flags, resource, dconn, NULL);
3701     else
3702         ret = doNativeMigrate(driver, vm, uri_out,
3703                               cookie, cookielen,
3704                               NULL, NULL, /* No out cookie with v2 migration */
3705                               flags, resource, dconn, NULL);
3706
3707     /* Perform failed. Make sure Finish doesn't overwrite the error */
3708     if (ret < 0)
3709         orig_err = virSaveLastError();
3710
3711     /* If Perform returns < 0, then we need to cancel the VM
3712      * startup on the destination
3713      */
3714     cancelled = ret < 0;
3715
3716  finish:
3717     /* In version 2 of the migration protocol, we pass the
3718      * status code from the sender to the destination host,
3719      * so it can do any cleanup if the migration failed.
3720      */
3721     dname = dname ? dname : vm->def->name;
3722     VIR_DEBUG("Finish2 %p ret=%d", dconn, ret);
3723     qemuDomainObjEnterRemote(vm);
3724     ddomain = dconn->driver->domainMigrateFinish2
3725         (dconn, dname, cookie, cookielen,
3726          uri_out ? uri_out : dconnuri, destflags, cancelled);
3727     qemuDomainObjExitRemote(vm);
3728     if (cancelled && ddomain)
3729         VIR_ERROR(_("finish step ignored that migration was cancelled"));
3730
3731  cleanup:
3732     if (ddomain) {
3733         virObjectUnref(ddomain);
3734         ret = 0;
3735     } else {
3736         ret = -1;
3737     }
3738
3739     virObjectUnref(st);
3740
3741     if (orig_err) {
3742         virSetError(orig_err);
3743         virFreeError(orig_err);
3744     }
3745     VIR_FREE(uri_out);
3746     VIR_FREE(cookie);
3747
3748     return ret;
3749 }
3750
3751
3752 /* This is essentially a re-impl of virDomainMigrateVersion3
3753  * from libvirt.c, but running in source libvirtd context,
3754  * instead of client app context & also adding in tunnel
3755  * handling */
3756 static int
3757 doPeer2PeerMigrate3(virQEMUDriverPtr driver,
3758                     virConnectPtr sconn,
3759                     virConnectPtr dconn,
3760                     const char *dconnuri,
3761                     virDomainObjPtr vm,
3762                     const char *xmlin,
3763                     const char *dname,
3764                     const char *uri,
3765                     const char *graphicsuri,
3766                     const char *listenAddress,
3767                     unsigned long long bandwidth,
3768                     bool useParams,
3769                     unsigned long flags)
3770 {
3771     virDomainPtr ddomain = NULL;
3772     char *uri_out = NULL;
3773     char *cookiein = NULL;
3774     char *cookieout = NULL;
3775     char *dom_xml = NULL;
3776     int cookieinlen = 0;
3777     int cookieoutlen = 0;
3778     int ret = -1;
3779     virErrorPtr orig_err = NULL;
3780     bool cancelled = true;
3781     virStreamPtr st = NULL;
3782     unsigned long destflags;
3783     virTypedParameterPtr params = NULL;
3784     int nparams = 0;
3785     int maxparams = 0;
3786
3787     VIR_DEBUG("driver=%p, sconn=%p, dconn=%p, dconnuri=%s, vm=%p, xmlin=%s, "
3788               "dname=%s, uri=%s, graphicsuri=%s, listenAddress=%s, "
3789               "bandwidth=%llu, useParams=%d, flags=%lx",
3790               driver, sconn, dconn, NULLSTR(dconnuri), vm, NULLSTR(xmlin),
3791               NULLSTR(dname), NULLSTR(uri), NULLSTR(graphicsuri),
3792               NULLSTR(listenAddress), bandwidth, useParams, flags);
3793
3794     /* Unlike the virDomainMigrateVersion3 counterpart, we don't need
3795      * to worry about auto-setting the VIR_MIGRATE_CHANGE_PROTECTION
3796      * bit here, because we are already running inside the context of
3797      * a single job.  */
3798
3799     dom_xml = qemuMigrationBeginPhase(driver, vm, xmlin, dname,
3800                                       &cookieout, &cookieoutlen, flags);
3801     if (!dom_xml)
3802         goto cleanup;
3803
3804     if (useParams) {
3805         if (virTypedParamsAddString(&params, &nparams, &maxparams,
3806                                     VIR_MIGRATE_PARAM_DEST_XML, dom_xml) < 0)
3807             goto cleanup;
3808
3809         if (dname &&
3810             virTypedParamsAddString(&params, &nparams, &maxparams,
3811                                     VIR_MIGRATE_PARAM_DEST_NAME, dname) < 0)
3812             goto cleanup;
3813
3814         if (uri &&
3815             virTypedParamsAddString(&params, &nparams, &maxparams,
3816                                     VIR_MIGRATE_PARAM_URI, uri) < 0)
3817             goto cleanup;
3818
3819         if (bandwidth &&
3820             virTypedParamsAddULLong(&params, &nparams, &maxparams,
3821                                     VIR_MIGRATE_PARAM_BANDWIDTH,
3822                                     bandwidth) < 0)
3823             goto cleanup;
3824
3825         if (graphicsuri &&
3826             virTypedParamsAddString(&params, &nparams, &maxparams,
3827                                     VIR_MIGRATE_PARAM_GRAPHICS_URI,
3828                                     graphicsuri) < 0)
3829             goto cleanup;
3830         if (listenAddress &&
3831             virTypedParamsAddString(&params, &nparams, &maxparams,
3832                                     VIR_MIGRATE_PARAM_LISTEN_ADDRESS,
3833                                     listenAddress) < 0)
3834             goto cleanup;
3835     }
3836
3837     if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED)
3838         flags |= VIR_MIGRATE_PAUSED;
3839
3840     destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR |
3841                           VIR_MIGRATE_AUTO_CONVERGE);
3842
3843     VIR_DEBUG("Prepare3 %p", dconn);
3844     cookiein = cookieout;
3845     cookieinlen = cookieoutlen;
3846     cookieout = NULL;
3847     cookieoutlen = 0;
3848     if (flags & VIR_MIGRATE_TUNNELLED) {
3849         if (!(st = virStreamNew(dconn, 0)))
3850             goto cleanup;
3851
3852         qemuDomainObjEnterRemote(vm);
3853         if (useParams) {
3854             ret = dconn->driver->domainMigratePrepareTunnel3Params
3855                 (dconn, st, params, nparams, cookiein, cookieinlen,
3856                  &cookieout, &cookieoutlen, destflags);
3857         } else {
3858             ret = dconn->driver->domainMigratePrepareTunnel3
3859                 (dconn, st, cookiein, cookieinlen, &cookieout, &cookieoutlen,
3860                  destflags, dname, bandwidth, dom_xml);
3861         }
3862         qemuDomainObjExitRemote(vm);
3863     } else {
3864         qemuDomainObjEnterRemote(vm);
3865         if (useParams) {
3866             ret = dconn->driver->domainMigratePrepare3Params
3867                 (dconn, params, nparams, cookiein, cookieinlen,
3868                  &cookieout, &cookieoutlen, &uri_out, destflags);
3869         } else {
3870             ret = dconn->driver->domainMigratePrepare3
3871                 (dconn, cookiein, cookieinlen, &cookieout, &cookieoutlen,
3872                  uri, &uri_out, destflags, dname, bandwidth, dom_xml);
3873         }
3874         qemuDomainObjExitRemote(vm);
3875     }
3876     VIR_FREE(dom_xml);
3877     if (ret == -1)
3878         goto cleanup;
3879
3880     if (flags & VIR_MIGRATE_OFFLINE) {
3881         VIR_DEBUG("Offline migration, skipping Perform phase");
3882         VIR_FREE(cookieout);
3883         cookieoutlen = 0;
3884         cancelled = false;
3885         goto finish;
3886     }
3887
3888     if (uri_out) {
3889         uri = uri_out;
3890         if (useParams &&
3891             virTypedParamsReplaceString(&params, &nparams,
3892                                         VIR_MIGRATE_PARAM_URI, uri_out) < 0) {
3893             orig_err = virSaveLastError();
3894             goto finish;
3895         }
3896     } else if (!uri && !(flags & VIR_MIGRATE_TUNNELLED)) {
3897         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
3898                        _("domainMigratePrepare3 did not set uri"));
3899         orig_err = virSaveLastError();
3900         goto finish;
3901     }
3902
3903     /* Perform the migration.  The driver isn't supposed to return
3904      * until the migration is complete. The src VM should remain
3905      * running, but in paused state until the destination can
3906      * confirm migration completion.
3907      */
3908     VIR_DEBUG("Perform3 %p uri=%s", sconn, NULLSTR(uri));
3909     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3);
3910     VIR_FREE(cookiein);
3911     cookiein = cookieout;
3912     cookieinlen = cookieoutlen;
3913     cookieout = NULL;
3914     cookieoutlen = 0;
3915     if (flags & VIR_MIGRATE_TUNNELLED) {
3916         ret = doTunnelMigrate(driver, vm, st,
3917                               cookiein, cookieinlen,
3918                               &cookieout, &cookieoutlen,
3919                               flags, bandwidth, dconn, graphicsuri);
3920     } else {
3921         ret = doNativeMigrate(driver, vm, uri,
3922                               cookiein, cookieinlen,
3923                               &cookieout, &cookieoutlen,
3924                               flags, bandwidth, dconn, graphicsuri);
3925     }
3926
3927     /* Perform failed. Make sure Finish doesn't overwrite the error */
3928     if (ret < 0) {
3929         orig_err = virSaveLastError();
3930     } else {
3931         qemuMigrationJobSetPhase(driver, vm,
3932                                  QEMU_MIGRATION_PHASE_PERFORM3_DONE);
3933     }
3934
3935     /* If Perform returns < 0, then we need to cancel the VM
3936      * startup on the destination
3937      */
3938     cancelled = ret < 0;
3939
3940  finish:
3941     /*
3942      * The status code from the source is passed to the destination.
3943      * The dest can cleanup in the source indicated it failed to
3944      * send all migration data. Returns NULL for ddomain if
3945      * the dest was unable to complete migration.
3946      */
3947     VIR_DEBUG("Finish3 %p ret=%d", dconn, ret);
3948     VIR_FREE(cookiein);
3949     cookiein = cookieout;
3950     cookieinlen = cookieoutlen;
3951     cookieout = NULL;
3952     cookieoutlen = 0;
3953
3954     if (useParams) {
3955         if (virTypedParamsGetString(params, nparams,
3956                                     VIR_MIGRATE_PARAM_DEST_NAME, NULL) <= 0 &&
3957             virTypedParamsReplaceString(&params, &nparams,
3958                                         VIR_MIGRATE_PARAM_DEST_NAME,
3959                                         vm->def->name) < 0) {
3960             ddomain = NULL;
3961         } else {
3962             qemuDomainObjEnterRemote(vm);
3963             ddomain = dconn->driver->domainMigrateFinish3Params
3964                 (dconn, params, nparams, cookiein, cookieinlen,
3965                  &cookieout, &cookieoutlen, destflags, cancelled);
3966             qemuDomainObjExitRemote(vm);
3967         }
3968     } else {
3969         dname = dname ? dname : vm->def->name;
3970         qemuDomainObjEnterRemote(vm);
3971         ddomain = dconn->driver->domainMigrateFinish3
3972             (dconn, dname, cookiein, cookieinlen, &cookieout, &cookieoutlen,
3973              dconnuri, uri, destflags, cancelled);
3974         qemuDomainObjExitRemote(vm);
3975     }
3976     if (cancelled && ddomain)
3977         VIR_ERROR(_("finish step ignored that migration was cancelled"));
3978
3979     /* If ddomain is NULL, then we were unable to start
3980      * the guest on the target, and must restart on the
3981      * source. There is a small chance that the ddomain
3982      * is NULL due to an RPC failure, in which case
3983      * ddomain could in fact be running on the dest.
3984      * The lock manager plugins should take care of
3985      * safety in this scenario.
3986      */
3987     cancelled = ddomain == NULL;
3988
3989     /* If finish3 set an error, and we don't have an earlier
3990      * one we need to preserve it in case confirm3 overwrites
3991      */
3992     if (!orig_err)
3993         orig_err = virSaveLastError();
3994
3995     /*
3996      * If cancelled, then src VM will be restarted, else
3997      * it will be killed
3998      */
3999     VIR_DEBUG("Confirm3 %p cancelled=%d vm=%p", sconn, cancelled, vm);
4000     VIR_FREE(cookiein);
4001     cookiein = cookieout;
4002     cookieinlen = cookieoutlen;
4003     cookieout = NULL;
4004     cookieoutlen = 0;
4005     ret = qemuMigrationConfirmPhase(driver, sconn, vm,
4006                                     cookiein, cookieinlen,
4007                                     flags, cancelled);
4008     /* If Confirm3 returns -1, there's nothing more we can
4009      * do, but fortunately worst case is that there is a
4010      * domain left in 'paused' state on source.
4011      */
4012     if (ret < 0)
4013         VIR_WARN("Guest %s probably left in 'paused' state on source",
4014                  vm->def->name);
4015
4016  cleanup:
4017     if (ddomain) {
4018         virObjectUnref(ddomain);
4019         ret = 0;
4020     } else {
4021         ret = -1;
4022     }
4023
4024     virObjectUnref(st);
4025
4026     if (orig_err) {
4027         virSetError(orig_err);
4028         virFreeError(orig_err);
4029     }
4030     VIR_FREE(uri_out);
4031     VIR_FREE(cookiein);
4032     VIR_FREE(cookieout);
4033     virTypedParamsFree(params, nparams);
4034     return ret;
4035 }
4036
4037
4038 static int virConnectCredType[] = {
4039     VIR_CRED_AUTHNAME,
4040     VIR_CRED_PASSPHRASE,
4041 };
4042
4043
4044 static virConnectAuth virConnectAuthConfig = {
4045     .credtype = virConnectCredType,
4046     .ncredtype = ARRAY_CARDINALITY(virConnectCredType),
4047 };
4048
4049
4050 static int doPeer2PeerMigrate(virQEMUDriverPtr driver,
4051                               virConnectPtr sconn,
4052                               virDomainObjPtr vm,
4053                               const char *xmlin,
4054                               const char *dconnuri,
4055                               const char *uri,
4056                               const char *graphicsuri,
4057                               const char *listenAddress,
4058                               unsigned long flags,
4059                               const char *dname,
4060                               unsigned long resource,
4061                               bool *v3proto)
4062 {
4063     int ret = -1;
4064     virConnectPtr dconn = NULL;
4065     bool p2p;
4066     virErrorPtr orig_err = NULL;
4067     bool offline = false;
4068     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
4069     bool useParams;
4070
4071     VIR_DEBUG("driver=%p, sconn=%p, vm=%p, xmlin=%s, dconnuri=%s, "
4072               "uri=%s, graphicsuri=%s, listenAddress=%s, flags=%lx, "
4073               "dname=%s, resource=%lu",
4074               driver, sconn, vm, NULLSTR(xmlin), NULLSTR(dconnuri),
4075               NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress),
4076               flags, NULLSTR(dname), resource);
4077
4078     /* the order of operations is important here; we make sure the
4079      * destination side is completely setup before we touch the source
4080      */
4081
4082     qemuDomainObjEnterRemote(vm);
4083     dconn = virConnectOpenAuth(dconnuri, &virConnectAuthConfig, 0);
4084     qemuDomainObjExitRemote(vm);
4085     if (dconn == NULL) {
4086         virReportError(VIR_ERR_OPERATION_FAILED,
4087                        _("Failed to connect to remote libvirt URI %s: %s"),
4088                        dconnuri, virGetLastErrorMessage());
4089         virObjectUnref(cfg);
4090         return -1;
4091     }
4092
4093     if (virConnectSetKeepAlive(dconn, cfg->keepAliveInterval,
4094                                cfg->keepAliveCount) < 0)
4095         goto cleanup;
4096
4097     qemuDomainObjEnterRemote(vm);
4098     p2p = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
4099                                    VIR_DRV_FEATURE_MIGRATION_P2P);
4100         /* v3proto reflects whether the caller used Perform3, but with
4101          * p2p migrate, regardless of whether Perform2 or Perform3
4102          * were used, we decide protocol based on what target supports
4103          */
4104     *v3proto = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
4105                                         VIR_DRV_FEATURE_MIGRATION_V3);
4106     useParams = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
4107                                          VIR_DRV_FEATURE_MIGRATION_PARAMS);
4108     if (flags & VIR_MIGRATE_OFFLINE)
4109         offline = VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
4110                                            VIR_DRV_FEATURE_MIGRATION_OFFLINE);
4111     qemuDomainObjExitRemote(vm);
4112
4113     if (!p2p) {
4114         virReportError(VIR_ERR_OPERATION_FAILED, "%s",
4115                        _("Destination libvirt does not support peer-to-peer migration protocol"));
4116         goto cleanup;
4117     }
4118
4119     /* Only xmlin, dname, uri, and bandwidth parameters can be used with
4120      * old-style APIs. */
4121     if (!useParams && graphicsuri) {
4122         virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
4123                        _("Migration APIs with extensible parameters are not "
4124                          "supported but extended parameters were passed"));
4125         goto cleanup;
4126     }
4127
4128     if (flags & VIR_MIGRATE_OFFLINE && !offline) {
4129         virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
4130                        _("offline migration is not supported by "
4131                          "the destination host"));
4132         goto cleanup;
4133     }
4134
4135     /* domain may have been stopped while we were talking to remote daemon */
4136     if (!virDomainObjIsActive(vm) && !(flags & VIR_MIGRATE_OFFLINE)) {
4137         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
4138                        _("guest unexpectedly quit"));
4139         goto cleanup;
4140     }
4141
4142     /* Change protection is only required on the source side (us), and
4143      * only for v3 migration when begin and perform are separate jobs.
4144      * But peer-2-peer is already a single job, and we still want to
4145      * talk to older destinations that would reject the flag.
4146      * Therefore it is safe to clear the bit here.  */
4147     flags &= ~VIR_MIGRATE_CHANGE_PROTECTION;
4148
4149     if (*v3proto) {
4150         ret = doPeer2PeerMigrate3(driver, sconn, dconn, dconnuri, vm, xmlin,
4151                                   dname, uri, graphicsuri, listenAddress,
4152                                   resource, useParams, flags);
4153     } else {
4154         ret = doPeer2PeerMigrate2(driver, sconn, dconn, vm,
4155                                   dconnuri, flags, dname, resource);
4156     }
4157
4158  cleanup:
4159     orig_err = virSaveLastError();
4160     qemuDomainObjEnterRemote(vm);
4161     virObjectUnref(dconn);
4162     qemuDomainObjExitRemote(vm);
4163     if (orig_err) {
4164         virSetError(orig_err);
4165         virFreeError(orig_err);
4166     }
4167     virObjectUnref(cfg);
4168     return ret;
4169 }
4170
4171
4172 /*
4173  * This implements perform part of the migration protocol when migration job
4174  * does not need to be active across several APIs, i.e., peer2peer migration or
4175  * perform phase of v2 non-peer2peer migration.
4176  */
4177 static int
4178 qemuMigrationPerformJob(virQEMUDriverPtr driver,
4179                         virConnectPtr conn,
4180                         virDomainObjPtr vm,
4181                         const char *xmlin,
4182                         const char *dconnuri,
4183                         const char *uri,
4184                         const char *graphicsuri,
4185                         const char *listenAddress,
4186                         const char *cookiein,
4187                         int cookieinlen,
4188                         char **cookieout,
4189                         int *cookieoutlen,
4190                         unsigned long flags,
4191                         const char *dname,
4192                         unsigned long resource,
4193                         bool v3proto)
4194 {
4195     virObjectEventPtr event = NULL;
4196     int ret = -1;
4197     virErrorPtr orig_err = NULL;
4198     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
4199     bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR);
4200
4201     if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
4202         goto cleanup;
4203
4204     if (!virDomainObjIsActive(vm) && !(flags & VIR_MIGRATE_OFFLINE)) {
4205         virReportError(VIR_ERR_OPERATION_INVALID,
4206                        "%s", _("domain is not running"));
4207         goto endjob;
4208     }
4209
4210     if (!qemuMigrationIsAllowed(driver, vm, NULL, true, abort_on_error))
4211         goto endjob;
4212
4213     if (!(flags & VIR_MIGRATE_UNSAFE) && !qemuMigrationIsSafe(vm->def))
4214         goto endjob;
4215
4216     qemuMigrationStoreDomainState(vm);
4217
4218     if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) {
4219         ret = doPeer2PeerMigrate(driver, conn, vm, xmlin,
4220                                  dconnuri, uri, graphicsuri, listenAddress,
4221                                  flags, dname, resource, &v3proto);
4222     } else {
4223         qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM2);
4224         ret = doNativeMigrate(driver, vm, uri, cookiein, cookieinlen,
4225                               cookieout, cookieoutlen,
4226                               flags, resource, NULL, NULL);
4227     }
4228     if (ret < 0)
4229         goto endjob;
4230
4231     /*
4232      * In v3 protocol, the source VM is not killed off until the
4233      * confirm step.
4234      */
4235     if (!v3proto) {
4236         qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_MIGRATED,
4237                         VIR_QEMU_PROCESS_STOP_MIGRATED);
4238         virDomainAuditStop(vm, "migrated");
4239         event = virDomainEventLifecycleNewFromObj(vm,
4240                                          VIR_DOMAIN_EVENT_STOPPED,
4241                                          VIR_DOMAIN_EVENT_STOPPED_MIGRATED);
4242     }
4243
4244  endjob:
4245     if (ret < 0)
4246         orig_err = virSaveLastError();
4247
4248     if (qemuMigrationRestoreDomainState(conn, vm)) {
4249         event = virDomainEventLifecycleNewFromObj(vm,
4250                                          VIR_DOMAIN_EVENT_RESUMED,
4251                                          VIR_DOMAIN_EVENT_RESUMED_MIGRATED);
4252     }
4253
4254     if (!qemuMigrationJobFinish(driver, vm)) {
4255         vm = NULL;
4256     } else if (!virDomainObjIsActive(vm) &&
4257                (!vm->persistent ||
4258                 (ret == 0 && (flags & VIR_MIGRATE_UNDEFINE_SOURCE)))) {
4259         if (flags & VIR_MIGRATE_UNDEFINE_SOURCE)
4260             virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm);
4261         qemuDomainRemoveInactive(driver, vm);
4262         vm = NULL;
4263     }
4264
4265     if (orig_err) {
4266         virSetError(orig_err);
4267         virFreeError(orig_err);
4268     }
4269
4270  cleanup:
4271     if (vm)
4272         virObjectUnlock(vm);
4273     if (event)
4274         qemuDomainEventQueue(driver, event);
4275     virObjectUnref(cfg);
4276     return ret;
4277 }
4278
4279 /*
4280  * This implements perform phase of v3 migration protocol.
4281  */
4282 static int
4283 qemuMigrationPerformPhase(virQEMUDriverPtr driver,
4284                           virConnectPtr conn,
4285                           virDomainObjPtr vm,
4286                           const char *uri,
4287                           const char *graphicsuri,
4288                           const char *cookiein,
4289                           int cookieinlen,
4290                           char **cookieout,
4291                           int *cookieoutlen,
4292                           unsigned long flags,
4293                           unsigned long resource)
4294 {
4295     virObjectEventPtr event = NULL;
4296     int ret = -1;
4297     bool hasrefs;
4298
4299     /* If we didn't start the job in the begin phase, start it now. */
4300     if (!(flags & VIR_MIGRATE_CHANGE_PROTECTION)) {
4301         if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
4302             goto cleanup;
4303     } else if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT)) {
4304         goto cleanup;
4305     }
4306
4307     qemuMigrationJobStartPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3);
4308     virCloseCallbacksUnset(driver->closeCallbacks, vm,
4309                            qemuMigrationCleanup);
4310
4311     ret = doNativeMigrate(driver, vm, uri, cookiein, cookieinlen,
4312                           cookieout, cookieoutlen,
4313                           flags, resource, NULL, graphicsuri);
4314
4315     if (ret < 0) {
4316         if (qemuMigrationRestoreDomainState(conn, vm)) {
4317             event = virDomainEventLifecycleNewFromObj(vm,
4318                                                       VIR_DOMAIN_EVENT_RESUMED,
4319                                                       VIR_DOMAIN_EVENT_RESUMED_MIGRATED);
4320         }
4321         goto endjob;
4322     }
4323
4324     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM3_DONE);
4325
4326     if (virCloseCallbacksSet(driver->closeCallbacks, vm, conn,
4327                              qemuMigrationCleanup) < 0)
4328         goto endjob;
4329
4330  endjob:
4331     if (ret < 0)
4332         hasrefs = qemuMigrationJobFinish(driver, vm);
4333     else
4334         hasrefs = qemuMigrationJobContinue(vm);
4335     if (!hasrefs) {
4336         vm = NULL;
4337     } else if (!virDomainObjIsActive(vm) && !vm->persistent) {
4338         qemuDomainRemoveInactive(driver, vm);
4339         vm = NULL;
4340     }
4341
4342  cleanup:
4343     if (vm)
4344         virObjectUnlock(vm);
4345     if (event)
4346         qemuDomainEventQueue(driver, event);
4347     return ret;
4348 }
4349
4350 int
4351 qemuMigrationPerform(virQEMUDriverPtr driver,
4352                      virConnectPtr conn,
4353                      virDomainObjPtr vm,
4354                      const char *xmlin,
4355                      const char *dconnuri,
4356                      const char *uri,
4357                      const char *graphicsuri,
4358                      const char *listenAddress,
4359                      const char *cookiein,
4360                      int cookieinlen,
4361                      char **cookieout,
4362                      int *cookieoutlen,
4363                      unsigned long flags,
4364                      const char *dname,
4365                      unsigned long resource,
4366                      bool v3proto)
4367 {
4368     VIR_DEBUG("driver=%p, conn=%p, vm=%p, xmlin=%s, dconnuri=%s, "
4369               "uri=%s, graphicsuri=%s, listenAddress=%s"
4370               "cookiein=%s, cookieinlen=%d, cookieout=%p, cookieoutlen=%p, "
4371               "flags=%lx, dname=%s, resource=%lu, v3proto=%d",
4372               driver, conn, vm, NULLSTR(xmlin), NULLSTR(dconnuri),
4373               NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress),
4374               NULLSTR(cookiein), cookieinlen, cookieout, cookieoutlen,
4375               flags, NULLSTR(dname), resource, v3proto);
4376
4377     if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) {
4378         if (cookieinlen) {
4379             virReportError(VIR_ERR_OPERATION_INVALID,
4380                            "%s", _("received unexpected cookie with P2P migration"));
4381             return -1;
4382         }
4383
4384         return qemuMigrationPerformJob(driver, conn, vm, xmlin, dconnuri, uri,
4385                                        graphicsuri, listenAddress,
4386                                        cookiein, cookieinlen,
4387                                        cookieout, cookieoutlen,
4388                                        flags, dname, resource, v3proto);
4389     } else {
4390         if (dconnuri) {
4391             virReportError(VIR_ERR_INTERNAL_ERROR,
4392                            "%s", _("Unexpected dconnuri parameter with non-peer2peer migration"));
4393             return -1;
4394         }
4395
4396         if (v3proto) {
4397             return qemuMigrationPerformPhase(driver, conn, vm, uri,
4398                                              graphicsuri,
4399                                              cookiein, cookieinlen,
4400                                              cookieout, cookieoutlen,
4401                                              flags, resource);
4402         } else {
4403             return qemuMigrationPerformJob(driver, conn, vm, xmlin, dconnuri,
4404                                            uri, graphicsuri, listenAddress,
4405                                            cookiein, cookieinlen,
4406                                            cookieout, cookieoutlen, flags,
4407                                            dname, resource, v3proto);
4408         }
4409     }
4410 }
4411
4412 static int
4413 qemuMigrationVPAssociatePortProfiles(virDomainDefPtr def)
4414 {
4415     size_t i;
4416     int last_good_net = -1;
4417     virDomainNetDefPtr net;
4418
4419     for (i = 0; i < def->nnets; i++) {
4420         net = def->nets[i];
4421         if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_DIRECT) {
4422             if (virNetDevVPortProfileAssociate(net->ifname,
4423                                                virDomainNetGetActualVirtPortProfile(net),
4424                                                &net->mac,
4425                                                virDomainNetGetActualDirectDev(net),
4426                                                -1,
4427                                                def->uuid,
4428                                                VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_FINISH,
4429                                                false) < 0) {
4430                 virReportError(VIR_ERR_OPERATION_FAILED,
4431                                _("Port profile Associate failed for %s"),
4432                                net->ifname);
4433                 goto err_exit;
4434             }
4435             VIR_DEBUG("Port profile Associate succeeded for %s", net->ifname);
4436
4437             if (virNetDevMacVLanVPortProfileRegisterCallback(net->ifname, &net->mac,
4438                                                              virDomainNetGetActualDirectDev(net), def->uuid,
4439                                                              virDomainNetGetActualVirtPortProfile(net),
4440                                                              VIR_NETDEV_VPORT_PROFILE_OP_CREATE))
4441                 goto err_exit;
4442         }
4443         last_good_net = i;
4444     }
4445
4446     return 0;
4447
4448  err_exit:
4449     for (i = 0; last_good_net != -1 && i < last_good_net; i++) {
4450         net = def->nets[i];
4451         if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_DIRECT) {
4452             ignore_value(virNetDevVPortProfileDisassociate(net->ifname,
4453                                                            virDomainNetGetActualVirtPortProfile(net),
4454                                                            &net->mac,
4455                                                            virDomainNetGetActualDirectDev(net),
4456                                                            -1,
4457                                                            VIR_NETDEV_VPORT_PROFILE_OP_MIGRATE_IN_FINISH));
4458         }
4459     }
4460     return -1;
4461 }
4462
4463
4464 virDomainPtr
4465 qemuMigrationFinish(virQEMUDriverPtr driver,
4466                     virConnectPtr dconn,
4467                     virDomainObjPtr vm,
4468                     const char *cookiein,
4469                     int cookieinlen,
4470                     char **cookieout,
4471                     int *cookieoutlen,
4472                     unsigned long flags,
4473                     int retcode,
4474                     bool v3proto)
4475 {
4476     virDomainPtr dom = NULL;
4477     virObjectEventPtr event = NULL;
4478     bool newVM = true;
4479     qemuMigrationCookiePtr mig = NULL;
4480     virErrorPtr orig_err = NULL;
4481     int cookie_flags = 0;
4482     qemuDomainObjPrivatePtr priv = vm->privateData;
4483     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
4484     virCapsPtr caps = NULL;
4485     unsigned short port;
4486
4487     VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, "
4488               "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d",
4489               driver, dconn, vm, NULLSTR(cookiein), cookieinlen,
4490               cookieout, cookieoutlen, flags, retcode);
4491
4492     port = priv->migrationPort;
4493     priv->migrationPort = 0;
4494
4495     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
4496         goto cleanup;
4497
4498     if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_IN))
4499         goto cleanup;
4500
4501     qemuMigrationJobStartPhase(driver, vm,
4502                                v3proto ? QEMU_MIGRATION_PHASE_FINISH3
4503                                        : QEMU_MIGRATION_PHASE_FINISH2);
4504
4505     qemuDomainCleanupRemove(vm, qemuMigrationPrepareCleanup);
4506
4507     cookie_flags = QEMU_MIGRATION_COOKIE_NETWORK;
4508     if (flags & VIR_MIGRATE_PERSIST_DEST)
4509         cookie_flags |= QEMU_MIGRATION_COOKIE_PERSISTENT;
4510
4511     if (!(mig = qemuMigrationEatCookie(driver, vm, cookiein,
4512                                        cookieinlen, cookie_flags)))
4513         goto endjob;
4514
4515     /* Did the migration go as planned?  If yes, return the domain
4516      * object, but if no, clean up the empty qemu process.
4517      */
4518     if (retcode == 0) {
4519         if (!virDomainObjIsActive(vm) && !(flags & VIR_MIGRATE_OFFLINE)) {
4520             virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
4521                            _("guest unexpectedly quit"));
4522             goto endjob;
4523         }
4524
4525         if (!(flags & VIR_MIGRATE_OFFLINE)) {
4526             if (qemuMigrationVPAssociatePortProfiles(vm->def) < 0) {
4527                 qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
4528                                 VIR_QEMU_PROCESS_STOP_MIGRATED);
4529                 virDomainAuditStop(vm, "failed");
4530                 event = virDomainEventLifecycleNewFromObj(vm,
4531                                                  VIR_DOMAIN_EVENT_STOPPED,
4532                                                  VIR_DOMAIN_EVENT_STOPPED_FAILED);
4533                 goto endjob;
4534             }
4535             if (mig->network)
4536                 if (qemuDomainMigrateOPDRelocate(driver, vm, mig) < 0)
4537                     VIR_WARN("unable to provide network data for relocation");
4538         }
4539
4540         qemuMigrationStopNBDServer(driver, vm, mig);
4541
4542         if (flags & VIR_MIGRATE_PERSIST_DEST) {
4543             virDomainDefPtr vmdef;
4544             if (vm->persistent)
4545                 newVM = false;
4546             vm->persistent = 1;
4547             if (mig->persistent)
4548                 vm->newDef = vmdef = mig->persistent;
4549             else
4550                 vmdef = virDomainObjGetPersistentDef(caps, driver->xmlopt, vm);
4551             if (!vmdef || virDomainSaveConfig(cfg->configDir, vmdef) < 0) {
4552                 /* Hmpf.  Migration was successful, but making it persistent
4553                  * was not.  If we report successful, then when this domain
4554                  * shuts down, management tools are in for a surprise.  On the
4555                  * other hand, if we report failure, then the management tools
4556                  * might try to restart the domain on the source side, even
4557                  * though the domain is actually running on the destination.
4558                  * Return a NULL dom pointer, and hope that this is a rare
4559                  * situation and management tools are smart.
4560                  */
4561
4562                 /*
4563                  * However, in v3 protocol, the source VM is still available
4564                  * to restart during confirm() step, so we kill it off now.
4565                  */
4566                 if (v3proto) {
4567                     if (!(flags & VIR_MIGRATE_OFFLINE)) {
4568                         qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
4569                                         VIR_QEMU_PROCESS_STOP_MIGRATED);
4570                         virDomainAuditStop(vm, "failed");
4571                     }
4572                     if (newVM)
4573                         vm->persistent = 0;
4574                 }
4575                 if (!vmdef)
4576                     virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
4577                                    _("can't get vmdef"));
4578                 goto endjob;
4579             }
4580
4581             event = virDomainEventLifecycleNewFromObj(vm,
4582                                              VIR_DOMAIN_EVENT_DEFINED,
4583                                              newVM ?
4584                                              VIR_DOMAIN_EVENT_DEFINED_ADDED :
4585                                              VIR_DOMAIN_EVENT_DEFINED_UPDATED);
4586             if (event)
4587                 qemuDomainEventQueue(driver, event);
4588             event = NULL;
4589         }
4590
4591         if (!(flags & VIR_MIGRATE_PAUSED) && !(flags & VIR_MIGRATE_OFFLINE)) {
4592             /* run 'cont' on the destination, which allows migration on qemu
4593              * >= 0.10.6 to work properly.  This isn't strictly necessary on
4594              * older qemu's, but it also doesn't hurt anything there
4595              */
4596             if (qemuProcessStartCPUs(driver, vm, dconn,
4597                                      VIR_DOMAIN_RUNNING_MIGRATED,
4598                                      QEMU_ASYNC_JOB_MIGRATION_IN) < 0) {
4599                 if (virGetLastError() == NULL)
4600                     virReportError(VIR_ERR_INTERNAL_ERROR,
4601                                    "%s", _("resume operation failed"));
4602                 /* Need to save the current error, in case shutting
4603                  * down the process overwrites it
4604                  */
4605                 orig_err = virSaveLastError();
4606
4607                 /*
4608                  * In v3 protocol, the source VM is still available to
4609                  * restart during confirm() step, so we kill it off
4610                  * now.
4611                  * In v2 protocol, the source is dead, so we leave
4612                  * target in paused state, in case admin can fix
4613                  * things up
4614                  */
4615                 if (v3proto) {
4616                     qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
4617                                     VIR_QEMU_PROCESS_STOP_MIGRATED);
4618                     virDomainAuditStop(vm, "failed");
4619                     event = virDomainEventLifecycleNewFromObj(vm,
4620                                                      VIR_DOMAIN_EVENT_STOPPED,
4621                                                      VIR_DOMAIN_EVENT_STOPPED_FAILED);
4622                 }
4623                 goto endjob;
4624             }
4625         }
4626
4627         dom = virGetDomain(dconn, vm->def->name, vm->def->uuid);
4628
4629         if (!(flags & VIR_MIGRATE_OFFLINE)) {
4630             event = virDomainEventLifecycleNewFromObj(vm,
4631                                              VIR_DOMAIN_EVENT_RESUMED,
4632                                              VIR_DOMAIN_EVENT_RESUMED_MIGRATED);
4633             if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED) {
4634                 virDomainObjSetState(vm, VIR_DOMAIN_PAUSED,
4635                                      VIR_DOMAIN_PAUSED_USER);
4636                 if (event)
4637                     qemuDomainEventQueue(driver, event);
4638                 event = virDomainEventLifecycleNewFromObj(vm,
4639                                                  VIR_DOMAIN_EVENT_SUSPENDED,
4640                                                  VIR_DOMAIN_EVENT_SUSPENDED_PAUSED);
4641             }
4642         }
4643
4644         if (virDomainObjIsActive(vm) &&
4645             virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0) {
4646             VIR_WARN("Failed to save status on vm %s", vm->def->name);
4647             goto endjob;
4648         }
4649
4650         /* Guest is successfully running, so cancel previous auto destroy */
4651         qemuProcessAutoDestroyRemove(driver, vm);
4652     } else if (!(flags & VIR_MIGRATE_OFFLINE)) {
4653         qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED,
4654                         VIR_QEMU_PROCESS_STOP_MIGRATED);
4655         virDomainAuditStop(vm, "failed");
4656         event = virDomainEventLifecycleNewFromObj(vm,
4657                                          VIR_DOMAIN_EVENT_STOPPED,
4658                                          VIR_DOMAIN_EVENT_STOPPED_FAILED);
4659     }
4660
4661     if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, 0) < 0)
4662         VIR_WARN("Unable to encode migration cookie");
4663
4664  endjob:
4665     if (qemuMigrationJobFinish(driver, vm) == 0) {
4666         vm = NULL;
4667     } else if (!vm->persistent && !virDomainObjIsActive(vm)) {
4668         qemuDomainRemoveInactive(driver, vm);
4669         vm = NULL;
4670     }
4671
4672  cleanup:
4673     virPortAllocatorRelease(driver->migrationPorts, port);
4674     if (vm) {
4675         VIR_FREE(priv->origname);
4676         virObjectUnlock(vm);
4677     }
4678     if (event)
4679         qemuDomainEventQueue(driver, event);
4680     qemuMigrationCookieFree(mig);
4681     if (orig_err) {
4682         virSetError(orig_err);
4683         virFreeError(orig_err);
4684     }
4685     virObjectUnref(caps);
4686     virObjectUnref(cfg);
4687     return dom;
4688 }
4689
4690
4691 /* Helper function called while vm is active.  */
4692 int
4693 qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm,
4694                     int fd, off_t offset, const char *path,
4695                     const char *compressor,
4696                     bool bypassSecurityDriver,
4697                     enum qemuDomainAsyncJob asyncJob)
4698 {
4699     qemuDomainObjPrivatePtr priv = vm->privateData;
4700     int rc;
4701     int ret = -1;
4702     bool restoreLabel = false;
4703     virCommandPtr cmd = NULL;
4704     int pipeFD[2] = { -1, -1 };
4705     unsigned long saveMigBandwidth = priv->migMaxBandwidth;
4706     char *errbuf = NULL;
4707     virErrorPtr orig_err = NULL;
4708
4709     /* Increase migration bandwidth to unlimited since target is a file.
4710      * Failure to change migration speed is not fatal. */
4711     if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
4712         qemuMonitorSetMigrationSpeed(priv->mon,
4713                                      QEMU_DOMAIN_MIG_BANDWIDTH_MAX);
4714         priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX;
4715         qemuDomainObjExitMonitor(driver, vm);
4716     }
4717
4718     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
4719         (!compressor || pipe(pipeFD) == 0)) {
4720         /* All right! We can use fd migration, which means that qemu
4721          * doesn't have to open() the file, so while we still have to
4722          * grant SELinux access, we can do it on fd and avoid cleanup
4723          * later, as well as skip futzing with cgroup.  */
4724         if (virSecurityManagerSetImageFDLabel(driver->securityManager, vm->def,
4725                                               compressor ? pipeFD[1] : fd) < 0)
4726             goto cleanup;
4727         bypassSecurityDriver = true;
4728     } else {
4729         /* Phooey - we have to fall back on exec migration, where qemu
4730          * has to popen() the file by name, and block devices have to be
4731          * given cgroup ACL permission.  We might also stumble on
4732          * a race present in some qemu versions where it does a wait()
4733          * that botches pclose.  */
4734         if (virCgroupHasController(priv->cgroup,
4735                                    VIR_CGROUP_CONTROLLER_DEVICES)) {
4736             int rv = virCgroupAllowDevicePath(priv->cgroup, path,
4737                                               VIR_CGROUP_DEVICE_RW);
4738             virDomainAuditCgroupPath(vm, priv->cgroup, "allow", path, "rw", rv == 0);
4739             if (rv == 1) {
4740                 /* path was not a device, no further need for cgroup */
4741             } else if (rv < 0) {
4742                 goto cleanup;
4743             }
4744         }
4745         if ((!bypassSecurityDriver) &&
4746             virSecurityManagerSetSavedStateLabel(driver->securityManager,
4747                                                  vm->def, path) < 0)
4748             goto cleanup;
4749         restoreLabel = true;
4750     }
4751
4752     if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
4753         goto cleanup;
4754
4755     if (!compressor) {
4756         const char *args[] = { "cat", NULL };
4757
4758         if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
4759             priv->monConfig->type == VIR_DOMAIN_CHR_TYPE_UNIX) {
4760             rc = qemuMonitorMigrateToFd(priv->mon,
4761                                         QEMU_MONITOR_MIGRATE_BACKGROUND,
4762                                         fd);
4763         } else {
4764             rc = qemuMonitorMigrateToFile(priv->mon,
4765                                           QEMU_MONITOR_MIGRATE_BACKGROUND,
4766                                           args, path, offset);
4767         }
4768     } else {
4769         const char *prog = compressor;
4770         const char *args[] = {
4771             prog,
4772             "-c",
4773             NULL
4774         };
4775         if (pipeFD[0] != -1) {
4776             cmd = virCommandNewArgs(args);
4777             virCommandSetInputFD(cmd, pipeFD[0]);
4778             virCommandSetOutputFD(cmd, &fd);
4779             virCommandSetErrorBuffer(cmd, &errbuf);
4780             virCommandDoAsyncIO(cmd);
4781             if (virSetCloseExec(pipeFD[1]) < 0) {
4782                 virReportSystemError(errno, "%s",
4783                                      _("Unable to set cloexec flag"));
4784                 qemuDomainObjExitMonitor(driver, vm);
4785                 goto cleanup;
4786             }
4787             if (virCommandRunAsync(cmd, NULL) < 0) {
4788                 qemuDomainObjExitMonitor(driver, vm);
4789                 goto cleanup;
4790             }
4791             rc = qemuMonitorMigrateToFd(priv->mon,
4792                                         QEMU_MONITOR_MIGRATE_BACKGROUND,
4793                                         pipeFD[1]);
4794             if (VIR_CLOSE(pipeFD[0]) < 0 ||
4795                 VIR_CLOSE(pipeFD[1]) < 0)
4796                 VIR_WARN("failed to close intermediate pipe");
4797         } else {
4798             rc = qemuMonitorMigrateToFile(priv->mon,
4799                                           QEMU_MONITOR_MIGRATE_BACKGROUND,
4800                                           args, path, offset);
4801         }
4802     }
4803     qemuDomainObjExitMonitor(driver, vm);
4804
4805     if (rc < 0)
4806         goto cleanup;
4807
4808     rc = qemuMigrationWaitForCompletion(driver, vm, asyncJob, NULL, false);
4809
4810     if (rc < 0) {
4811         if (rc == -2) {
4812             orig_err = virSaveLastError();
4813             virCommandAbort(cmd);
4814             if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
4815                 qemuMonitorMigrateCancel(priv->mon);
4816                 qemuDomainObjExitMonitor(driver, vm);
4817             }
4818         }
4819         goto cleanup;
4820     }
4821
4822     if (cmd && virCommandWait(cmd, NULL) < 0)
4823         goto cleanup;
4824
4825     ret = 0;
4826
4827  cleanup:
4828     if (ret < 0 && !orig_err)
4829         orig_err = virSaveLastError();
4830
4831     /* Restore max migration bandwidth */
4832     if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) {
4833         qemuMonitorSetMigrationSpeed(priv->mon, saveMigBandwidth);
4834         priv->migMaxBandwidth = saveMigBandwidth;
4835         qemuDomainObjExitMonitor(driver, vm);
4836     }
4837
4838     VIR_FORCE_CLOSE(pipeFD[0]);
4839     VIR_FORCE_CLOSE(pipeFD[1]);
4840     if (cmd) {
4841         VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf));
4842         VIR_FREE(errbuf);
4843         virCommandFree(cmd);
4844     }
4845     if (restoreLabel && (!bypassSecurityDriver) &&
4846         virSecurityManagerRestoreSavedStateLabel(driver->securityManager,
4847                                                  vm->def, path) < 0)
4848         VIR_WARN("failed to restore save state label on %s", path);
4849
4850     if (virCgroupHasController(priv->cgroup,
4851                                VIR_CGROUP_CONTROLLER_DEVICES)) {
4852         int rv = virCgroupDenyDevicePath(priv->cgroup, path,
4853                                          VIR_CGROUP_DEVICE_RWM);
4854         virDomainAuditCgroupPath(vm, priv->cgroup, "deny", path, "rwm", rv == 0);
4855     }
4856
4857     if (orig_err) {
4858         virSetError(orig_err);
4859         virFreeError(orig_err);
4860     }
4861
4862     return ret;
4863 }
4864
4865 int
4866 qemuMigrationJobStart(virQEMUDriverPtr driver,
4867                       virDomainObjPtr vm,
4868                       enum qemuDomainAsyncJob job)
4869 {
4870     qemuDomainObjPrivatePtr priv = vm->privateData;
4871
4872     if (qemuDomainObjBeginAsyncJob(driver, vm, job) < 0)
4873         return -1;
4874
4875     if (job == QEMU_ASYNC_JOB_MIGRATION_IN) {
4876         qemuDomainObjSetAsyncJobMask(vm, QEMU_JOB_NONE);
4877     } else {
4878         qemuDomainObjSetAsyncJobMask(vm, DEFAULT_JOB_MASK |
4879                                      JOB_MASK(QEMU_JOB_SUSPEND) |
4880                                      JOB_MASK(QEMU_JOB_MIGRATION_OP));
4881     }
4882
4883     priv->job.info.type = VIR_DOMAIN_JOB_UNBOUNDED;
4884
4885     return 0;
4886 }
4887
4888 void
4889 qemuMigrationJobSetPhase(virQEMUDriverPtr driver,
4890                          virDomainObjPtr vm,
4891                          enum qemuMigrationJobPhase phase)
4892 {
4893     qemuDomainObjPrivatePtr priv = vm->privateData;
4894
4895     if (phase < priv->job.phase) {
4896         VIR_ERROR(_("migration protocol going backwards %s => %s"),
4897                   qemuMigrationJobPhaseTypeToString(priv->job.phase),
4898                   qemuMigrationJobPhaseTypeToString(phase));
4899         return;
4900     }
4901
4902     qemuDomainObjSetJobPhase(driver, vm, phase);
4903 }
4904
4905 void
4906 qemuMigrationJobStartPhase(virQEMUDriverPtr driver,
4907                            virDomainObjPtr vm,
4908                            enum qemuMigrationJobPhase phase)
4909 {
4910     virObjectRef(vm);
4911     qemuMigrationJobSetPhase(driver, vm, phase);
4912 }
4913
4914 bool
4915 qemuMigrationJobContinue(virDomainObjPtr vm)
4916 {
4917     qemuDomainObjReleaseAsyncJob(vm);
4918     return virObjectUnref(vm);
4919 }
4920
4921 bool
4922 qemuMigrationJobIsActive(virDomainObjPtr vm,
4923                          enum qemuDomainAsyncJob job)
4924 {
4925     qemuDomainObjPrivatePtr priv = vm->privateData;
4926
4927     if (priv->job.asyncJob != job) {
4928         const char *msg;
4929
4930         if (job == QEMU_ASYNC_JOB_MIGRATION_IN)
4931             msg = _("domain '%s' is not processing incoming migration");
4932         else
4933             msg = _("domain '%s' is not being migrated");
4934
4935         virReportError(VIR_ERR_OPERATION_INVALID, msg, vm->def->name);
4936         return false;
4937     }
4938     return true;
4939 }
4940
4941 bool
4942 qemuMigrationJobFinish(virQEMUDriverPtr driver, virDomainObjPtr vm)
4943 {
4944     return qemuDomainObjEndAsyncJob(driver, vm);
4945 }